1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include <cstdint>
16 #include <initializer_list>
17 #include <vector>
18
19 #include <gmock/gmock.h>
20 #include <gtest/gtest.h>
21 #include "flatbuffers/flatbuffers.h" // from @flatbuffers
22 #include "tensorflow/lite/kernels/test_util.h"
23 #include "tensorflow/lite/schema/schema_generated.h"
24
25 namespace tflite {
26 namespace {
27
28 using ::testing::ElementsAreArray;
29
30 class QuantizedLSTMOpModel : public MultiOpModel {
31 public:
QuantizedLSTMOpModel(int numBatches,int inputSize,float weightsScale,int32_t weightsZeroPoint,int outputSize,std::initializer_list<uint8_t> weights,std::initializer_list<int32_t> biases,bool prepend_noop)32 QuantizedLSTMOpModel(int numBatches, int inputSize, float weightsScale,
33 int32_t weightsZeroPoint, int outputSize,
34 std::initializer_list<uint8_t> weights,
35 std::initializer_list<int32_t> biases,
36 // If true the LTSM node will be preceded by a noop
37 // one (add to 0)
38 bool prepend_noop) {
39 std::vector<uint32_t> inputs;
40
41 input_size_ = inputSize;
42 output_size_ = outputSize;
43 prepend_noop_ = prepend_noop;
44
45 std::vector<int> input_shape{numBatches, inputSize};
46 std::vector<int> output_shape{numBatches, outputSize};
47 std::vector<int> weight_shape{4 * outputSize, outputSize + inputSize};
48 std::vector<int> state_shape{numBatches, outputSize};
49 std::vector<int> bias_shape{4 * outputSize};
50
51 std::vector<int> lstm_inputs;
52
53 const TensorData input_tensor_data{
54 TensorType_UINT8, input_shape, 0.0f, 0.0f, 1. / 128., 128};
55
56 if (prepend_noop) {
57 zero_input_ = AddInput(input_tensor_data);
58 } else {
59 zero_input_ = 0;
60 }
61
62 input_ = AddInput(input_tensor_data);
63
64 prev_output_ = AddVariableInput(
65 {TensorType_UINT8, output_shape, 0.0f, 0.0f, 1. / 128., 128});
66 // Biases and Weights have to be constant in order to allow NNAPI
67 // delegation
68 weights_ = AddConstInput<uint8_t>({TensorType_UINT8, weight_shape, 0.0f,
69 0.0f, weightsScale, weightsZeroPoint},
70 weights);
71 biases_ = AddConstInput<int32_t>(
72 {TensorType_INT32, bias_shape, 0.0f, 0.0f, weightsScale / 128, 0},
73 biases);
74 prev_cell_state_ = AddVariableInput(
75 {TensorType_INT16, state_shape, 0.0f, 0.0f, 1. / 2048., 0});
76
77 sum_out_ = AddOutput(input_tensor_data);
78
79 output_ =
80 AddOutput({TensorType_UINT8, output_shape, 0.0f, 0.0f, 1. / 128., 128});
81 cell_state_out_ =
82 AddOutput({TensorType_INT16, state_shape, 0.0f, 0.0f, 1. / 2048., 0});
83 output_concat_temp_ =
84 AddOutput({TensorType_UINT8, output_shape, 0.0f, 0.0f, 1. / 128., 128});
85 output_activation_temp_ =
86 AddOutput({TensorType_INT16, output_shape, 0.0f, 0.0f, 1. / 128., 128});
87
88 if (prepend_noop) {
89 AddBuiltinOp(
90 BuiltinOperator_ADD, BuiltinOptions_AddOptions,
91 CreateAddOptions(builder_, ActivationFunctionType_NONE).Union(),
92 {zero_input_, input_}, {sum_out_});
93
94 lstm_inputs.push_back(sum_out_);
95 } else {
96 lstm_inputs.push_back(input_);
97 }
98
99 lstm_inputs.push_back(prev_output_);
100 lstm_inputs.push_back(weights_);
101 lstm_inputs.push_back(biases_);
102 lstm_inputs.push_back(prev_cell_state_);
103
104 std::vector<int> lstm_outputs{output_, cell_state_out_, output_concat_temp_,
105 output_activation_temp_};
106
107 AddBuiltinOp(BuiltinOperator_LSTM, BuiltinOptions_LSTMOptions,
108 CreateLSTMOptions(builder_, ActivationFunctionType_TANH, 0.0,
109 0.0, LSTMKernelType_BASIC)
110 .Union(),
111 lstm_inputs, lstm_outputs);
112
113 if (prepend_noop) {
114 BuildInterpreter({GetShape(input_), GetShape(zero_input_),
115 GetShape(prev_output_), GetShape(weights_),
116 GetShape(biases_), GetShape(prev_cell_state_)});
117 } else {
118 BuildInterpreter({GetShape(input_), GetShape(prev_output_),
119 GetShape(weights_), GetShape(biases_),
120 GetShape(prev_cell_state_)});
121 }
122 // init feedback inputs to zero
123 std::vector<int16_t> initial_state(GetTensorSize(cell_state_out_), 0);
124 PopulateTensor(prev_cell_state_, initial_state);
125 std::vector<uint8_t> initial_prev_output(GetTensorSize(output_), 0);
126 PopulateTensor(prev_output_, initial_prev_output);
127 }
128
inputSize()129 int inputSize() { return input_size_; }
130
outputSize()131 int outputSize() { return output_size_; }
132
setInput(const std::vector<uint8_t> & input)133 void setInput(const std::vector<uint8_t>& input) {
134 PopulateTensor(input_, input);
135 if (prepend_noop_) {
136 std::vector<uint8_t> zero(GetTensorSize(zero_input_), 128);
137 PopulateTensor(zero_input_, zero);
138 }
139 }
140
getOutput()141 std::vector<uint8_t> getOutput() { return ExtractVector<uint8_t>(output_); }
142
143 private:
144 // Inputs
145 int input_;
146 int weights_;
147 int biases_;
148 int prev_cell_state_;
149 int prev_output_;
150 // Outputs
151 int cell_state_out_;
152 int output_;
153 int output_concat_temp_;
154 int output_activation_temp_;
155
156 int input_size_;
157 int output_size_;
158 bool prepend_noop_;
159 int zero_input_;
160 int sum_out_;
161 };
162
163 class QuantizedLstmTest : public ::testing::Test,
164 public testing::WithParamInterface<bool> {
165 protected:
VerifyGoldens(const std::vector<std::vector<uint8_t>> & input,const std::vector<std::vector<uint8_t>> & output,QuantizedLSTMOpModel * lstm)166 void VerifyGoldens(const std::vector<std::vector<uint8_t>>& input,
167 const std::vector<std::vector<uint8_t>>& output,
168 QuantizedLSTMOpModel* lstm) {
169 const int numBatches = input.size();
170 ASSERT_GT(numBatches, 0);
171 const int inputSize = lstm->inputSize();
172 ASSERT_GT(inputSize, 0);
173 const int inputSequenceSize = input[0].size() / inputSize;
174 ASSERT_GT(inputSequenceSize, 0);
175 for (int i = 0; i < inputSequenceSize; ++i) {
176 std::vector<uint8_t> inputStep;
177 for (int b = 0; b < numBatches; ++b) {
178 const uint8_t* batchStart = input[b].data() + i * inputSize;
179 const uint8_t* batchEnd = batchStart + inputSize;
180 inputStep.insert(inputStep.end(), batchStart, batchEnd);
181 }
182 lstm->setInput(inputStep);
183 ASSERT_EQ(lstm->Invoke(), kTfLiteOk);
184
185 const int outputSize = lstm->outputSize();
186 std::vector<float> expected;
187 for (int b = 0; b < numBatches; ++b) {
188 const uint8_t* goldenBatchStart = output[b].data() + i * outputSize;
189 const uint8_t* goldenBatchEnd = goldenBatchStart + outputSize;
190 expected.insert(expected.end(), goldenBatchStart, goldenBatchEnd);
191 }
192 EXPECT_THAT(lstm->getOutput(), ElementsAreArray(expected));
193 }
194 }
195 };
196
197 // Inputs and weights in this test are random and the test only checks that the
198 // outputs are equal to outputs obtained from running TF Lite version of
199 // quantized LSTM on the same inputs.
TEST_P(QuantizedLstmTest,BasicQuantizedLstmTest)200 TEST_P(QuantizedLstmTest, BasicQuantizedLstmTest) {
201 const int numBatches = 2;
202 const int inputSize = 2;
203 const int outputSize = 4;
204
205 float weightsScale = 0.00408021;
206 int weightsZeroPoint = 100;
207
208 bool prepend_dummy_node = GetParam();
209
210 QuantizedLSTMOpModel lstm(
211 numBatches, inputSize, weightsScale, weightsZeroPoint, outputSize,
212
213 // This data are copied from QuantizedLSTMTest.cpp in NNAPI source code
214 // I have to recompose the weight matrix before passing it to the model
215
216 // recurrentToInputWeights inputToInputWeights
217 {254, 206, 77, 168, 146, 250, 71, 20, 215, 6, 235, 171, 223, 7, 118, 225,
218 10, 218, 59, 130, 174, 26, 171, 108,
219
220 // recurrentToCellWeights inputToCellWeights
221 172, 60, 205, 65, 133, 34, 14, 0, 140, 168, 29, 49, 240, 223, 133, 56,
222 206, 109, 142, 64, 246, 216, 54, 183,
223
224 // recurrentToForgetWeights inputToForgetWeights
225 137, 240, 103, 52, 24, 50, 68, 51, 237, 112, 132, 179, 0, 220, 89, 23,
226 158, 110, 69, 4, 207, 253, 3, 169,
227
228 // recurrentToOutputWeights inputToOutputWeights
229 106, 214, 67, 23, 195, 187, 59, 158, 45, 3, 11, 99, 119, 132, 49, 205,
230 109, 10, 129, 218, 11, 98, 218, 48},
231
232 // inputGateBias
233 {-7876, 13488, -726, 32839,
234 // cellGateBias
235 39481, 48624, 48976, -21419,
236 // forgetGateBias
237 9206, -46884, -11693, -38724,
238 // outputGateBias
239 -58999, -17050, -41852, -40538},
240 prepend_dummy_node);
241 // clang-format on
242
243 // LSTM input is stored as numBatches x (sequenceLength x inputSize) vector.
244 std::vector<std::vector<uint8_t>> lstmInput;
245 // clang-format off
246 lstmInput = {{154, 166,
247 166, 179,
248 141, 141},
249 {100, 200,
250 50, 150,
251 111, 222}};
252 // clang-format on
253
254 // LSTM output is stored as numBatches x (sequenceLength x outputSize) vector.
255 std::vector<std::vector<uint8_t>> lstmGoldenOutput;
256 /*
257 This is the output used in NNAPI's QuantizedLSTMTest.cpp
258 I get slightly different values that are consistent running with or
259 without acceleration
260
261 lstmGoldenOutput = {{136, 150, 140, 115,
262 140, 151, 146, 112,
263 139, 153, 146, 114},
264 {135, 152, 138, 112,
265 136, 156, 142, 112,
266 141, 154, 146, 108}};
267 */
268
269 // clang-format off
270 lstmGoldenOutput = {{131, 152, 136, 109,
271 138, 150, 145, 111,
272 139, 152, 146, 113},
273 {131, 153, 135, 107,
274 134, 154, 140, 111,
275 140, 154, 145, 108}};
276 // clang-format on
277 VerifyGoldens(lstmInput, lstmGoldenOutput, &lstm);
278 }
279
280 INSTANTIATE_TEST_SUITE_P(QuantizedLstmTest, QuantizedLstmTest,
281 testing::Values(false, true));
282
283 } // namespace
284 } // namespace tflite
285