1# Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Generate a series of TensorFlow graphs that become tflite test cases. 16 17Usage: 18 19generate_examples <output directory> 20 21bazel run //tensorflow/lite/testing:generate_examples 22 23To more easily debug failures use (or override) the --save_graphdefs flag to 24place text proto graphdefs into the generated zip files. 25""" 26 27import copy 28import datetime 29import os 30import re 31import zipfile 32 33import tensorflow.compat.v1 as tf 34 35# TODO(aselle): Disable GPU for now 36os.environ["CUDA_VISIBLE_DEVICES"] = "-1" 37 38# pylint: disable=g-import-not-at-top 39# pylint: disable=g-multiple-import 40# pylint: disable=unused-import 41from tensorflow.lite.testing.op_tests.abs import make_abs_tests 42from tensorflow.lite.testing.op_tests.add_n import make_add_n_tests 43from tensorflow.lite.testing.op_tests.arg_min_max import make_arg_min_max_tests 44from tensorflow.lite.testing.op_tests.atan2 import make_atan2_tests 45from tensorflow.lite.testing.op_tests.batch_to_space_nd import make_batch_to_space_nd_tests 46from tensorflow.lite.testing.op_tests.batchmatmul import make_batchmatmul_tests 47from tensorflow.lite.testing.op_tests.binary_op import make_add_tests, make_div_tests, make_sub_tests, make_mul_tests, make_pow_tests, make_floor_div_tests, make_floor_mod_tests, make_squared_difference_tests 48from tensorflow.lite.testing.op_tests.broadcast_args import make_broadcast_args_tests 49from tensorflow.lite.testing.op_tests.broadcast_gradient_args import make_broadcast_gradient_args_tests 50from tensorflow.lite.testing.op_tests.broadcast_to import make_broadcast_to_tests 51from tensorflow.lite.testing.op_tests.cast import make_cast_tests 52from tensorflow.lite.testing.op_tests.ceil import make_ceil_tests 53from tensorflow.lite.testing.op_tests.complex_abs import make_complex_abs_tests 54from tensorflow.lite.testing.op_tests.concat import make_concat_tests 55from tensorflow.lite.testing.op_tests.cond import make_cond_tests 56from tensorflow.lite.testing.op_tests.constant import make_constant_tests 57from tensorflow.lite.testing.op_tests.control_dep import make_control_dep_tests 58from tensorflow.lite.testing.op_tests.conv import make_conv_tests 59from tensorflow.lite.testing.op_tests.conv2d_transpose import make_conv2d_transpose_tests 60from tensorflow.lite.testing.op_tests.conv3d import make_conv3d_tests 61from tensorflow.lite.testing.op_tests.conv3d_transpose import make_conv3d_transpose_tests 62from tensorflow.lite.testing.op_tests.conv_activation import make_conv_relu_tests, make_conv_relu1_tests, make_conv_relu6_tests 63from tensorflow.lite.testing.op_tests.conv_bias_activation import make_conv_bias_relu6_tests 64from tensorflow.lite.testing.op_tests.conv_to_depthwiseconv_with_shared_weights import make_conv_to_depthwiseconv_with_shared_weights_tests 65from tensorflow.lite.testing.op_tests.conv_with_shared_weights import make_conv_with_shared_weights_tests 66from tensorflow.lite.testing.op_tests.cos import make_cos_tests 67from tensorflow.lite.testing.op_tests.cumsum import make_cumsum_tests 68# Placeholder for make_dense_image_warp_tests import 69from tensorflow.lite.testing.op_tests.depth_to_space import make_depth_to_space_tests 70from tensorflow.lite.testing.op_tests.depthwiseconv import make_depthwiseconv_tests 71from tensorflow.lite.testing.op_tests.dynamic_rnn import make_dynamic_rnn_tests 72from tensorflow.lite.testing.op_tests.dynamic_update_slice import make_dynamic_update_slice_tests 73from tensorflow.lite.testing.op_tests.einsum import make_einsum_tests 74from tensorflow.lite.testing.op_tests.elementwise import make_sin_tests, make_log_tests, make_sqrt_tests, make_rsqrt_tests, make_square_tests 75from tensorflow.lite.testing.op_tests.elu import make_elu_tests 76from tensorflow.lite.testing.op_tests.embedding_lookup import make_embedding_lookup_tests 77from tensorflow.lite.testing.op_tests.equal import make_equal_tests 78from tensorflow.lite.testing.op_tests.exp import make_exp_tests 79from tensorflow.lite.testing.op_tests.expand_dims import make_expand_dims_tests 80from tensorflow.lite.testing.op_tests.expm1 import make_expm1_tests 81from tensorflow.lite.testing.op_tests.eye import make_eye_tests 82from tensorflow.lite.testing.op_tests.fill import make_fill_tests 83from tensorflow.lite.testing.op_tests.floor import make_floor_tests 84from tensorflow.lite.testing.op_tests.fully_connected import make_fully_connected_tests 85from tensorflow.lite.testing.op_tests.fused_batch_norm import make_fused_batch_norm_tests 86from tensorflow.lite.testing.op_tests.gather import make_gather_tests 87from tensorflow.lite.testing.op_tests.gather_nd import make_gather_nd_tests 88from tensorflow.lite.testing.op_tests.gather_with_constant import make_gather_with_constant_tests 89from tensorflow.lite.testing.op_tests.gelu import make_gelu_tests 90from tensorflow.lite.testing.op_tests.global_batch_norm import make_global_batch_norm_tests 91from tensorflow.lite.testing.op_tests.greater import make_greater_tests 92from tensorflow.lite.testing.op_tests.greater_equal import make_greater_equal_tests 93from tensorflow.lite.testing.op_tests.hardswish import make_hardswish_tests 94from tensorflow.lite.testing.op_tests.identify_dilated_conv import make_identify_dilated_conv_tests 95from tensorflow.lite.testing.op_tests.identify_dilated_conv1d import make_identify_dilated_conv1d_tests 96from tensorflow.lite.testing.op_tests.identity import make_identity_tests 97from tensorflow.lite.testing.op_tests.imag import make_imag_tests 98from tensorflow.lite.testing.op_tests.irfft2d import make_irfft2d_tests 99from tensorflow.lite.testing.op_tests.is_finite import make_is_finite_tests 100from tensorflow.lite.testing.op_tests.l2norm import make_l2norm_tests 101# Note: This is a regression test for a bug (b/122651451) that Toco incorrectly 102# erases the reduction indices array while it's shared with other ops. 103from tensorflow.lite.testing.op_tests.l2norm_shared_epsilon import make_l2norm_shared_epsilon_tests 104from tensorflow.lite.testing.op_tests.leaky_relu import make_leaky_relu_tests 105from tensorflow.lite.testing.op_tests.less import make_less_tests 106from tensorflow.lite.testing.op_tests.less_equal import make_less_equal_tests 107from tensorflow.lite.testing.op_tests.local_response_norm import make_local_response_norm_tests 108from tensorflow.lite.testing.op_tests.log_softmax import make_log_softmax_tests 109from tensorflow.lite.testing.op_tests.logic import make_logical_or_tests, make_logical_and_tests, make_logical_xor_tests 110from tensorflow.lite.testing.op_tests.lstm import make_lstm_tests 111from tensorflow.lite.testing.op_tests.matrix_diag import make_matrix_diag_tests 112from tensorflow.lite.testing.op_tests.matrix_set_diag import make_matrix_set_diag_tests 113from tensorflow.lite.testing.op_tests.max_pool_with_argmax import make_max_pool_with_argmax_tests 114from tensorflow.lite.testing.op_tests.maximum import make_maximum_tests 115from tensorflow.lite.testing.op_tests.minimum import make_minimum_tests 116from tensorflow.lite.testing.op_tests.mirror_pad import make_mirror_pad_tests 117from tensorflow.lite.testing.op_tests.multinomial import make_multinomial_tests 118from tensorflow.lite.testing.op_tests.nearest_upsample import make_nearest_upsample_tests 119from tensorflow.lite.testing.op_tests.neg import make_neg_tests 120from tensorflow.lite.testing.op_tests.not_equal import make_not_equal_tests 121from tensorflow.lite.testing.op_tests.one_hot import make_one_hot_tests 122from tensorflow.lite.testing.op_tests.pack import make_pack_tests 123from tensorflow.lite.testing.op_tests.pad import make_pad_tests 124from tensorflow.lite.testing.op_tests.padv2 import make_padv2_tests 125from tensorflow.lite.testing.op_tests.parse_example import make_parse_example_tests 126from tensorflow.lite.testing.op_tests.placeholder_with_default import make_placeholder_with_default_tests 127from tensorflow.lite.testing.op_tests.pool import make_l2_pool_tests, make_avg_pool_tests, make_max_pool_tests 128from tensorflow.lite.testing.op_tests.pool3d import make_avg_pool3d_tests 129from tensorflow.lite.testing.op_tests.pool3d import make_max_pool3d_tests 130from tensorflow.lite.testing.op_tests.prelu import make_prelu_tests 131from tensorflow.lite.testing.op_tests.random_standard_normal import make_random_standard_normal_tests 132from tensorflow.lite.testing.op_tests.random_uniform import make_random_uniform_tests 133from tensorflow.lite.testing.op_tests.range import make_range_tests 134from tensorflow.lite.testing.op_tests.rank import make_rank_tests 135from tensorflow.lite.testing.op_tests.real import make_real_tests 136from tensorflow.lite.testing.op_tests.reciprocal import make_reciprocal_tests 137from tensorflow.lite.testing.op_tests.reduce import make_mean_tests, make_sum_tests, make_reduce_prod_tests, make_reduce_max_tests, make_reduce_min_tests, make_reduce_any_tests, make_reduce_all_tests 138from tensorflow.lite.testing.op_tests.relu import make_relu_tests 139from tensorflow.lite.testing.op_tests.relu1 import make_relu1_tests 140from tensorflow.lite.testing.op_tests.relu6 import make_relu6_tests 141from tensorflow.lite.testing.op_tests.reshape import make_reshape_tests 142from tensorflow.lite.testing.op_tests.resize_bilinear import make_resize_bilinear_tests 143from tensorflow.lite.testing.op_tests.resize_nearest_neighbor import make_resize_nearest_neighbor_tests 144# For verifying https://github.com/tensorflow/tensorflow/issues/23599 145from tensorflow.lite.testing.op_tests.resolve_constant_strided_slice import make_resolve_constant_strided_slice_tests 146from tensorflow.lite.testing.op_tests.reverse_sequence import make_reverse_sequence_tests 147from tensorflow.lite.testing.op_tests.reverse_v2 import make_reverse_v2_tests 148from tensorflow.lite.testing.op_tests.rfft import make_rfft_tests 149from tensorflow.lite.testing.op_tests.rfft2d import make_rfft2d_tests 150from tensorflow.lite.testing.op_tests.roll import make_roll_tests 151from tensorflow.lite.testing.op_tests.roll import make_roll_with_constant_tests 152from tensorflow.lite.testing.op_tests.round import make_round_tests 153from tensorflow.lite.testing.op_tests.scatter_nd import make_scatter_nd_tests 154from tensorflow.lite.testing.op_tests.segment_sum import make_segment_sum_tests 155from tensorflow.lite.testing.op_tests.shape import make_shape_tests 156from tensorflow.lite.testing.op_tests.shape_to_strided_slice import make_shape_to_strided_slice_tests 157from tensorflow.lite.testing.op_tests.sigmoid import make_sigmoid_tests 158from tensorflow.lite.testing.op_tests.slice import make_slice_tests 159from tensorflow.lite.testing.op_tests.softmax import make_softmax_tests 160from tensorflow.lite.testing.op_tests.softplus import make_softplus_tests 161from tensorflow.lite.testing.op_tests.space_to_batch_nd import make_space_to_batch_nd_tests 162from tensorflow.lite.testing.op_tests.space_to_depth import make_space_to_depth_tests 163from tensorflow.lite.testing.op_tests.sparse_to_dense import make_sparse_to_dense_tests 164from tensorflow.lite.testing.op_tests.split import make_split_tests 165from tensorflow.lite.testing.op_tests.splitv import make_splitv_tests 166from tensorflow.lite.testing.op_tests.squeeze import make_squeeze_tests 167from tensorflow.lite.testing.op_tests.squeeze_transpose import make_squeeze_transpose_tests 168from tensorflow.lite.testing.op_tests.static_hashtable import make_static_hashtable_tests 169from tensorflow.lite.testing.op_tests.static_rnn_with_control_flow_v2 import make_static_rnn_with_control_flow_v2_tests 170from tensorflow.lite.testing.op_tests.stft import make_stft_tests 171from tensorflow.lite.testing.op_tests.strided_slice import make_strided_slice_tests, make_strided_slice_1d_exhaustive_tests 172from tensorflow.lite.testing.op_tests.strided_slice_np_style import make_strided_slice_np_style_tests 173from tensorflow.lite.testing.op_tests.tanh import make_tanh_tests 174from tensorflow.lite.testing.op_tests.tensor_list_concat import make_tensor_list_concat_tests 175from tensorflow.lite.testing.op_tests.tensor_list_dynamic_shape import make_tensor_list_dynamic_shape_tests 176from tensorflow.lite.testing.op_tests.tensor_list_get_item import make_tensor_list_get_item_tests 177from tensorflow.lite.testing.op_tests.tensor_list_length import make_tensor_list_length_tests 178from tensorflow.lite.testing.op_tests.tensor_list_resize import make_tensor_list_resize_tests 179from tensorflow.lite.testing.op_tests.tensor_list_set_item import make_tensor_list_set_item_tests 180from tensorflow.lite.testing.op_tests.tensor_scatter_add import make_tensor_scatter_add_tests 181from tensorflow.lite.testing.op_tests.tensor_scatter_update import make_tensor_scatter_update_tests 182from tensorflow.lite.testing.op_tests.tile import make_tile_tests 183from tensorflow.lite.testing.op_tests.topk import make_topk_tests 184from tensorflow.lite.testing.op_tests.transpose import make_transpose_tests 185from tensorflow.lite.testing.op_tests.transpose_conv import make_transpose_conv_tests 186from tensorflow.lite.testing.op_tests.unfused_gru import make_unfused_gru_tests 187from tensorflow.lite.testing.op_tests.unique import make_unique_tests 188from tensorflow.lite.testing.op_tests.unpack import make_unpack_tests 189from tensorflow.lite.testing.op_tests.unroll_batch_matmul import make_unroll_batch_matmul_tests 190from tensorflow.lite.testing.op_tests.unsorted_segment import make_unsorted_segment_max_tests, make_unsorted_segment_min_tests, make_unsorted_segment_prod_tests, make_unsorted_segment_sum_tests 191from tensorflow.lite.testing.op_tests.where import make_where_tests 192from tensorflow.lite.testing.op_tests.where_v2 import make_where_v2_tests 193from tensorflow.lite.testing.op_tests.while_loop import make_while_tests 194from tensorflow.lite.testing.op_tests.zeros_like import make_zeros_like_tests 195from tensorflow.lite.testing.zip_test_utils import get_test_function 196 197 198class MultiGenState: 199 """State of multiple set generation process. 200 201 This state class stores the information needed when generating the examples 202 for multiple test set. The stored informations are open archive object to be 203 shared, information on test target for current iteration of generation, 204 accumulated generation results. 205 """ 206 207 def __init__(self): 208 # Open archive. 209 self.archive = None 210 # Test name for current generation. 211 self.test_name = None 212 # Label base path containing the test name. 213 # Each of the test data path in the zip archive is derived from this path. 214 # If this path is "a/b/c/d.zip", an example of generated test data path 215 # is "a/b/c/d_input_type=tf.float32,input_shape=[2,2].inputs". 216 # The test runner interpretes the test name of this path as "d". 217 # Label base path also should finish with ".zip". 218 self.label_base_path = None 219 # Zip manifests. 220 self.zip_manifest = [] 221 # Number of all parameters accumulated. 222 self.parameter_count = 0 223 224 225class Options: 226 """All options for example generation.""" 227 228 def __init__(self): 229 # Directory where the outputs will be go. 230 self.output_path = None 231 # Particular zip to output. 232 self.zip_to_output = None 233 # If a particular model is affected by a known bug count it as a converter 234 # error. 235 self.known_bugs_are_errors = False 236 # Raise an exception if any converter error is encountered. 237 self.ignore_converter_errors = False 238 # Include intermediate graphdefs in the output zip files. 239 self.save_graphdefs = False 240 # Whether the TFLite Flex converter is being used. 241 self.run_with_flex = False 242 # Whether to generate test cases for edgetpu. 243 self.make_edgetpu_tests = False 244 # Whether to generate test cases for TF PTQ. 245 self.make_tf_ptq_tests = False 246 # For TF Quantization only: where conversion for HLO target. 247 self.hlo_aware_conversion = True 248 # The function to convert a TensorFLow model to TFLite model. 249 # See the document for `mlir_convert` function for its required signature. 250 self.tflite_convert_function = None 251 # A map from regular expression to bug number. Any test failure with label 252 # matching the expression will be considered due to the corresponding bug. 253 self.known_bugs = {} 254 # Make tests by setting TF forward compatibility horizon to the future. 255 self.make_forward_compat_test = False 256 # No limitation on the number of tests. 257 self.no_tests_limit = False 258 # Do not create conversion report. 259 self.no_conversion_report = False 260 # State of multiple test set generation. This stores state values those 261 # should be kept and updated while generating examples over multiple 262 # test sets. 263 # TODO(juhoha): Separate the state from the options. 264 self.multi_gen_state = None 265 self.mlir_quantizer = False 266 # The list of ops' name that should exist in the converted model. 267 # This feature is currently only supported in MLIR conversion path. 268 # Example of supported ops' name: 269 # - "AVERAGE_POOL_2D" for builtin op. 270 # - "NumericVerify" for custom op. 271 self.expected_ops_in_converted_model = [] 272 # Whether to skip generating tests with high dimension input shape. 273 self.skip_high_dimension_inputs = False 274 # Whether to enable DynamicUpdateSlice op. 275 self.enable_dynamic_update_slice = False 276 # Whether to disable unrolling batch matmul. 277 self.disable_batchmatmul_unfold = False 278 279 280def _prepare_dir(options): 281 282 def mkdir_if_not_exist(x): 283 if not os.path.isdir(x): 284 os.mkdir(x) 285 if not os.path.isdir(x): 286 raise RuntimeError("Failed to create dir %r" % x) 287 288 opstest_path = os.path.join(options.output_path) 289 mkdir_if_not_exist(opstest_path) 290 291 292def generate_examples(options): 293 """Generate examples for a test set. 294 295 Args: 296 options: Options containing information to generate examples. 297 298 Raises: 299 RuntimeError: if the test function cannot be found. 300 """ 301 _prepare_dir(options) 302 303 out = options.zip_to_output 304 # Some zip filenames contain a postfix identifying the conversion mode. The 305 # list of valid conversion modes is defined in 306 # generated_test_conversion_modes() in build_def.bzl. 307 308 if options.multi_gen_state: 309 test_name = options.multi_gen_state.test_name 310 else: 311 # Remove suffixes to extract the test name from the output name. 312 test_name = re.sub( 313 r"(_(|with-flex|forward-compat|edgetpu|mlir-quant))?(_xnnpack)?\.zip$", 314 "", 315 out, 316 count=1) 317 318 test_function_name = "make_%s_tests" % test_name 319 test_function = get_test_function(test_function_name) 320 if test_function is None: 321 raise RuntimeError("Can't find a test function to create %r. Tried %r" % 322 (out, test_function_name)) 323 if options.make_forward_compat_test: 324 future_date = datetime.date.today() + datetime.timedelta(days=30) 325 with tf.compat.forward_compatibility_horizon(future_date.year, 326 future_date.month, 327 future_date.day): 328 test_function(options) 329 else: 330 test_function(options) 331 332 333def generate_multi_set_examples(options, test_sets): 334 """Generate examples for test sets. 335 336 Args: 337 options: Options containing information to generate examples. 338 test_sets: List of the name of test sets to generate examples. 339 """ 340 _prepare_dir(options) 341 342 multi_gen_state = MultiGenState() 343 options.multi_gen_state = multi_gen_state 344 345 zip_path = os.path.join(options.output_path, options.zip_to_output) 346 with zipfile.PyZipFile(zip_path, "w") as archive: 347 multi_gen_state.archive = archive 348 349 for test_name in test_sets: 350 # Some generation function can change the value of the options object. 351 # To keep the original options for each run, we use shallow copy. 352 new_options = copy.copy(options) 353 354 # Remove suffix and set test_name to run proper test generation function. 355 multi_gen_state.test_name = re.sub( 356 r"(_(|with-flex|forward-compat|mlir-quant))?$", 357 "", 358 test_name, 359 count=1) 360 # Set label base path to write test data files with proper path. 361 multi_gen_state.label_base_path = os.path.join( 362 os.path.dirname(zip_path), test_name + ".zip") 363 364 generate_examples(new_options) 365 366 zipinfo = zipfile.ZipInfo("manifest.txt") 367 archive.writestr(zipinfo, "".join(multi_gen_state.zip_manifest), 368 zipfile.ZIP_DEFLATED) 369