1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/lite/testing/generate_testspec.h"
17
18 #include <iostream>
19 #include <random>
20 #include <string>
21 #include <utility>
22
23 #include "tensorflow/core/framework/types.h"
24 #include "tensorflow/lite/testing/join.h"
25 #include "tensorflow/lite/testing/split.h"
26 #include "tensorflow/lite/testing/tf_driver.h"
27 #include "tensorflow/lite/testing/tflite_driver.h"
28
29 namespace tflite {
30 namespace testing {
31 namespace {
32
33 // Generates input name / value pairs according to given shape and distribution.
34 // Fills `out` with a pair of string, which the first element is input name and
35 // the second element is comma separated values in string.
36 template <typename T, typename RandomEngine, typename RandomDistribution>
GenerateCsv(const string & name,const std::vector<int> & shape,RandomEngine * engine,RandomDistribution distribution,std::pair<string,string> * out)37 void GenerateCsv(const string& name, const std::vector<int>& shape,
38 RandomEngine* engine, RandomDistribution distribution,
39 std::pair<string, string>* out) {
40 std::vector<T> data =
41 GenerateRandomTensor<T>(shape, [&]() { return distribution(*engine); });
42 *out = std::make_pair(name, Join(data.data(), data.size(), ","));
43 }
44
45 // Generates random values for `input_layer` according to given value types and
46 // shapes.
47 // Fills `out` with a vector of string pairs, which the first element in the
48 // pair is the input name from `input_layer` and the second element is comma
49 // separated values in string.
50 template <typename RandomEngine>
GenerateInputValues(RandomEngine * engine,const std::vector<string> & input_layer,const std::vector<string> & input_layer_type,const std::vector<string> & input_layer_shape)51 std::vector<std::pair<string, string>> GenerateInputValues(
52 RandomEngine* engine, const std::vector<string>& input_layer,
53 const std::vector<string>& input_layer_type,
54 const std::vector<string>& input_layer_shape) {
55 std::vector<std::pair<string, string>> input_values;
56 input_values.resize(input_layer.size());
57 for (int i = 0; i < input_layer.size(); i++) {
58 tensorflow::DataType type;
59 CHECK(DataTypeFromString(input_layer_type[i], &type));
60 auto shape = Split<int>(input_layer_shape[i], ",");
61 const auto& name = input_layer[i];
62
63 switch (type) {
64 case tensorflow::DT_FLOAT:
65 GenerateCsv<float>(name, shape, engine,
66 std::uniform_real_distribution<float>(-0.5, 0.5),
67 &input_values[i]);
68 break;
69 case tensorflow::DT_UINT8:
70 GenerateCsv<uint8_t>(name, shape, engine,
71 std::uniform_int_distribution<uint32_t>(0, 255),
72 &input_values[i]);
73 break;
74 case tensorflow::DT_INT32:
75 GenerateCsv<int32_t>(name, shape, engine,
76 std::uniform_int_distribution<int32_t>(-100, 100),
77 &input_values[i]);
78 break;
79 case tensorflow::DT_INT64:
80 GenerateCsv<int64_t>(name, shape, engine,
81 std::uniform_int_distribution<int64_t>(-100, 100),
82 &input_values[i]);
83 break;
84 case tensorflow::DT_BOOL:
85 GenerateCsv<int>(name, shape, engine,
86 std::uniform_int_distribution<int>(0, 1),
87 &input_values[i]);
88 break;
89 default:
90 fprintf(stderr, "Unsupported type %d (%s) when generating testspec.\n",
91 type, input_layer_type[i].c_str());
92 input_values.clear();
93 return input_values;
94 }
95 }
96 return input_values;
97 }
98
GenerateTestSpecFromRunner(std::iostream & stream,int num_invocations,const std::vector<string> & input_layer,const std::vector<string> & input_layer_type,const std::vector<string> & input_layer_shape,const std::vector<string> & output_layer,TestRunner * runner)99 bool GenerateTestSpecFromRunner(std::iostream& stream, int num_invocations,
100 const std::vector<string>& input_layer,
101 const std::vector<string>& input_layer_type,
102 const std::vector<string>& input_layer_shape,
103 const std::vector<string>& output_layer,
104 TestRunner* runner) {
105 auto input_size = input_layer.size();
106 if (input_layer_shape.size() != input_size ||
107 input_layer_type.size() != input_size) {
108 fprintf(stderr,
109 "Input size not match. Expected %lu, got %lu input types, %lu "
110 "input shapes.\n",
111 input_size, input_layer_type.size(), input_layer_shape.size());
112 return false;
113 }
114
115 stream << "reshape {\n";
116 for (int i = 0; i < input_size; i++) {
117 const auto& name = input_layer[i];
118 const auto& shape = input_layer_shape[i];
119 stream << " input { key: \"" << name << "\" value: \"" << shape
120 << "\" }\n";
121 }
122 stream << "}\n";
123
124 // Generate inputs.
125 std::mt19937 random_engine;
126 for (int i = 0; i < num_invocations; ++i) {
127 // Note that the input values are random, so each invocation will have a
128 // different set.
129 auto input_values = GenerateInputValues(
130 &random_engine, input_layer, input_layer_type, input_layer_shape);
131 if (input_values.empty()) {
132 std::cerr << "Unable to generate input values for the TensorFlow model. "
133 "Make sure the correct values are defined for "
134 "input_layer, input_layer_type, and input_layer_shape."
135 << std::endl;
136 return false;
137 }
138
139 // Run TensorFlow.
140 runner->Invoke(input_values);
141 if (!runner->IsValid()) {
142 std::cerr << runner->GetErrorMessage() << std::endl;
143 return false;
144 }
145
146 // Write second part of test spec, with inputs and outputs.
147 stream << "invoke {\n";
148 for (const auto& entry : input_values) {
149 stream << " input { key: \"" << entry.first << "\" value: \""
150 << entry.second << "\" }\n";
151 }
152 for (const auto& name : output_layer) {
153 stream << " output { key: \"" << name << "\" value: \""
154 << runner->ReadOutput(name) << "\" }\n";
155 if (!runner->IsValid()) {
156 std::cerr << runner->GetErrorMessage() << std::endl;
157 return false;
158 }
159 }
160 stream << "}\n";
161 }
162
163 return true;
164 }
165
166 } // namespace
167
GenerateTestSpecFromTensorflowModel(std::iostream & stream,const string & tensorflow_model_path,const string & tflite_model_path,int num_invocations,const std::vector<string> & input_layer,const std::vector<string> & input_layer_type,const std::vector<string> & input_layer_shape,const std::vector<string> & output_layer)168 bool GenerateTestSpecFromTensorflowModel(
169 std::iostream& stream, const string& tensorflow_model_path,
170 const string& tflite_model_path, int num_invocations,
171 const std::vector<string>& input_layer,
172 const std::vector<string>& input_layer_type,
173 const std::vector<string>& input_layer_shape,
174 const std::vector<string>& output_layer) {
175 CHECK_EQ(input_layer.size(), input_layer_type.size());
176 CHECK_EQ(input_layer.size(), input_layer_shape.size());
177
178 // Invoke tensorflow model.
179 TfDriver runner(input_layer, input_layer_type, input_layer_shape,
180 output_layer);
181 if (!runner.IsValid()) {
182 std::cerr << runner.GetErrorMessage() << std::endl;
183 return false;
184 }
185
186 runner.LoadModel(tensorflow_model_path);
187 if (!runner.IsValid()) {
188 std::cerr << runner.GetErrorMessage() << std::endl;
189 return false;
190 }
191 // Write first part of test spec, defining model and input shapes.
192 stream << "load_model: " << tflite_model_path << "\n";
193 return GenerateTestSpecFromRunner(stream, num_invocations, input_layer,
194 input_layer_type, input_layer_shape,
195 output_layer, &runner);
196 }
197
GenerateTestSpecFromTFLiteModel(std::iostream & stream,const string & tflite_model_path,int num_invocations,const std::vector<string> & input_layer,const std::vector<string> & input_layer_type,const std::vector<string> & input_layer_shape,const std::vector<string> & output_layer)198 bool GenerateTestSpecFromTFLiteModel(
199 std::iostream& stream, const string& tflite_model_path, int num_invocations,
200 const std::vector<string>& input_layer,
201 const std::vector<string>& input_layer_type,
202 const std::vector<string>& input_layer_shape,
203 const std::vector<string>& output_layer) {
204 TfLiteDriver runner;
205 runner.LoadModel(tflite_model_path);
206 if (!runner.IsValid()) {
207 std::cerr << runner.GetErrorMessage() << std::endl;
208 return false;
209 }
210 runner.AllocateTensors();
211 return GenerateTestSpecFromRunner(stream, num_invocations, input_layer,
212 input_layer_type, input_layer_shape,
213 output_layer, &runner);
214 }
215
216 } // namespace testing
217 } // namespace tflite
218