1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h" 16 17 namespace tflite { 18 19 const constexpr char* NnapiAccelerationTestParams::kAccelerationTestConfig = 20 R"( 21 ## Every Test can be allowlisted or denylisted using a regexp on its test_id 22 23 ## Test_id 24 # 25 # The test_id is test_suite_name / test_name, this differs from the 26 # name used by the build because of the / separator instead of . 27 # Parameterized tests names are composed by the base test name / test / ordinal 28 # the ordinal is the position in the list of parameters generated by the 29 # cardinal product of all the different parameter sets 30 31 # Denylist/Allowlist 32 # To denylist an element simply add - before the test_id regex 33 34 ## Rules evaluation 35 # 36 # Rules are checked in order, the first matching completes the browsing 37 # This can be useful to put more specific rules first and generic default 38 # ones below 39 40 ## Test Arguments 41 # 42 # The test can be parameterized with the minimum Android SDK version 43 # to apply the acceleration validation for. 44 # If omitted will use 27 45 46 #test-id,min-android-sdk-version 47 48 # activations_test 49 FloatActivationsOpTest/Softmax[13]D,29 50 QuantizedActivationsOpTest/Softmax[13]D.+nt8,29 51 FloatActivationsOpTest/Softmax\dD 52 QuantizedActivationsOpTest/Softmax\dD.+nt8 53 FloatActivationsOpTest/LogSoftmax,29 54 FloatActivationsOpTest/PRelu,29 55 LogisticOpTest/LogisticOpTest/Sigmoid(.+nt8)?/\d+ 56 LogisticOpTest/LogisticOpTest/Sigmoid/\d+ 57 TanhOpTest/TanhOpTest/Tanh(.+nt8)?/\d+,29 58 FloatActivationsOpTest/Elu,30 59 FloatActivationsOpTest/Relu 60 FloatActivationsOpTest/Relu1 61 FloatActivationsOpTest/Relu6 62 FloatActivationsOpTest/HardSwish 63 FloatActivationsOpTest/LeakyRelu,29 64 QuantizedActivationsOpTest/HardSwish 65 QuantizedActivationsOpTest/HardSwishBias 66 -QuantizedActivationsOpTest/Relu.?Int16 67 QuantizedActivationsOpTest/Relu.* 68 -QuantizedActivationsOpTest/LeakyReluInt16,30 69 QuantizedActivationsOpTest/LeakyRelu.*,30 70 QuantizedActivationsOpTest/Relu.+nt8 71 QuantizedActivationsOpTest/PRelu,29 72 QuantizedActivationsOpTest/PReluSameShapes,29 73 QuantizedActivationsOpTest/PReluInt8.+,30 74 PReluOpTest/.*,29 75 76 77 # add_test 78 FloatAddOpModel/.+ 79 QuantizedAddOpModel/QuantizedTestsNoActivation.+nt8 80 QuantizedAddOpModel/QuantizedVariousInputShapes.+ 81 QuantizedAddOpModel/QuantizedWithScalarBroadcast.+nt8 82 QuantizedAddOpModel/QuantizedWithMixedBroadcas.+nt8 83 84 # arg_min_max_test 85 # Only tests with ConstantAxis && OutputType == TensorType_INT32 86 # (element 4 and 6 in the test parameter list) 87 # Supported only from NNAPI 1.2 88 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgFloat/[46],29 89 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+Arg.+nt8/[46],29 90 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgInt/[46],29 91 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgMulDimensions/[46],29 92 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgNegativeAxis/[46],29 93 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgOutput64/[46],29 94 95 # basic_rnn_test 96 RnnOpTest/BlackBoxTest 97 98 # batch_to_space_nd_test 99 BatchToSpaceNDOpTest/SimpleConstTest.* 100 BatchToSpaceNDOpTest/BatchOneConstTest.* 101 102 # bidirectional_sequence_lstm_test 103 # Only test with non quantized weights 104 LSTMOpTest/LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping/0,29 105 # Only test with non quantized weights 106 LSTMOpTest/LSTMOpTest/BlackBoxTestMergedOutput/0,29 107 LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClippingReverse,29 108 LSTMOpTest/BlackBoxTestWithCifgWithPeepholeNoProjectionNoClipping,29 109 LSTMOpTest/BlackBoxTestWithCifgWithPeepholeNoProjectionNoClippingReversed,29 110 LSTMOpTest/BlackBoxTestWithPeepholeWithProjectionNoClipping,29 111 LSTMOpTest/BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor,29 112 # Only test with non quantized weights 113 LSTMOpTest/LSTMOpTest/BlackBoxTestWithAuxInputZeroAuxWeight/0,29 114 QuantizationOrNot/LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping/0,29 115 QuantizationOrNot/LSTMOpTest/BlackBoxTestMergedOutput/0,29 116 QuantizationOrNot/LSTMOpTest/BlackBoxTestWithAuxInputZeroAuxWeight/0,29 117 LSTMOpTest/LSTMOpTest/BlackBoxTestWithAuxInput/0,29 118 119 # cast_test 120 CastOpModel/CastFloatToIn32 121 CastOpModel/CastInt32ToFloat,29 122 CastOpModel/CastFloatToUInt8,29 123 CastOpModel/CastUInt8ToFloat,29 124 CastOpModel/CastInt32ToUInt8,29 125 CastOpModel/CastUInt8ToInt32,29 126 127 # comparisons_test 128 ComparisonsTest/.+,29 129 130 # concatenation_test 131 ConcatenationOpTest/ThreeDimensionalOneInput 132 ConcatenationOpTest/OneTrivialInput 133 ConcatenationOpTest/TwoDimensionalOneInput 134 ConcatenationOpTest/TwoInputsTwoAxesNegativeAxes 135 ConcatenationOpTest/TwoInputsTwoAxesNegativeAxesNonQuantized 136 ConcatenationOpTest/FourInputs 137 ConcatenationOpTest/FourInputsQuantizedUint8 138 ConcatenationOpTest/FourInputsQuantizedInt8 139 ConcatenationOpTest/ThreeDimensionalNonQuantizedOneInput 140 ConcatenationOpTest/OneTrivialNonQuantizedInput 141 ConcatenationOpTest/TwoDimensionalNonQuantizedOneInput 142 ConcatenationOpTest/FourInputsQuantizedMixedRange,29 143 ConcatenationOpTest/FourInputsQuantizedMixedRangeClampingLogic,29 144 145 # conv_test 146 -ConvolutionOpTest/ConvolutionOpTest.SimplePerTensorTest/.+ 147 ConvolutionOpTest/ConvolutionOpTest.SimpleTestFloatWithDilation/.+,29 148 ConvolutionOpTest/ConvolutionOpTest.SimpleTestLargeIrregularQuantized/.+,29 149 ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedOutputMultiplierGreaterThan1/.+,29 150 ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedWithDilation/.+,29 151 ConvolutionOpTest/ConvolutionOpTest.SimplePerChannelTest/.+,29 152 ConvolutionOpTest/ConvolutionOpTest/.+Hybrid.+,29/ 153 ConvolutionOpTest/ConvolutionOpTest/.+/\d+ 154 155 156 # depthwise_conv_test 157 .+ConvolutionOpTest/.+/\d+,29 158 159 # dequantize_test 160 DequantizeOpTest/Uint8 161 DequantizeOpTest/Int8,30 162 163 # depth_to_space_test 164 DepthToSpaceOpModel/Float32 165 DepthToSpaceOpModel/UInt8 166 DepthToSpaceOpModel/int8 167 168 # div_test 169 -FloatDivOpTest/WithBroadcast5D 170 FloatDivOpTest/.+ 171 172 # elementwise_test 173 ElementWise/Abs,29 174 ElementWise/Sin,29 175 ElementWise/Log,29 176 ElementWise/Sqrt,29 177 ElementWise/Rsqrt,29 178 ElementWise/LogicalNot,29 179 180 # embedding_lookup_test 181 EmbeddingLookupOpTest/SimpleTest 182 183 # exp_test 184 ExpOpTest/FloatTest,29 185 186 # expand_dims_test 187 # Only constant tensors models 188 ExpandDimsOpTest/.+/1,29 189 190 # fill_test 191 FillOpTest/FillOpTest/FillInt32/0,30 192 FillOpTest/FillOpTest/FillFloat/0,30 193 FillOpTest/FillOpTest/FillFloatInt32Dims/0,30 194 195 # floor_test 196 FloorOpTest/.+ 197 198 # fully_connected_test 199 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest/\d+ 200 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest2/\d+ 201 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTestQuantized.+nt8/\d+,29 202 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTestSingleBatchQuantizedInt8/\d+,29 203 QuantizedFullyConnectedOpTest/SimpleTestQuantizedOutputMultiplierGreaterThan1Uint8/\d+,29 204 QuantizedFullyConnectedOpTest/SimpleTestQuantizedOutputMultiplierGreaterThan1Int8/\d+,29 205 HybridFullyConnectedOpTest/SimpleTestQuantizedUint8,29 206 HybridFullyConnectedOpTest/SimpleTestQuantizedInt8,29 207 HybridAsymmetricInputFullyConnectedOpTest.SimpleTestQuantizedUint8,29 208 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest4DInput/\d+ 209 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTest4dInputQuantizedUint8/\d+ 210 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTest4dInputQuantizedOutputMultiplierGreaterThan1Uint8/\d+,29 211 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/BlackBoxTest/\d+ 212 FloatFullyConnectedOpTest/SimpleTestNoBias 213 214 # gather_test 215 GatherOpTest/Shuffle,29 216 GatherOpTest/Test1DInput1DIndex,29 217 GatherOpTest/Test2DIndexWith2DResult,29 218 FloatGatherOpTest/Duplicate,29 219 FloatGatherOpTest/Slice,29 220 FloatGatherOpTest/Axis1,29 221 FloatGatherOpTest/Axis1Slice,29 222 FloatGatherOpTest/LastAxis,29 223 TypesGatherOpTest/Float32Int32,29 224 TypesGatherOpTest/Int32Int32,29 225 TypesGatherOpTest/Uint8Int32,29 226 TypesGatherOpTest/Int8Int32,29 227 -TypesGatherOpTest/.*Int16.* 228 229 # hashtable_lookup_test 230 # All test excepted the string one should be accelerated 231 -HashtableLookupOpTest/TestString 232 HashtableLookupOpTest/.+ 233 234 # l2norm_test 235 L2NormOpTest/.+,29 236 237 # local_response_norm_test 238 LocalResponseNormOpTest/.+ 239 240 # logical_test 241 LogicalTest/.+,29 242 243 # lsh_projection_test 244 -LSHProjectionOpTest2/Sparse3DInputs 245 LSHProjectionOpTest2/Sparse1DInputs,29 246 LSHProjectionOpTest2/.+ 247 248 # Before the lstm because of clashing with matchers 249 # unidirectional_sequence_lstm_test 250 NoCifgNoPeepholeNoProjectionNoClippingUnidirectionalLstmTest/LstmBlackBoxTest,29 251 CifgPeepholeNoProjectionNoClippingUnidirectionalLstmTest/NonLayerNormLstmBlackBoxTest,29 252 # Only the two tests above, disabling all possible matches from the lstm tests 253 # coming after 254 -.+UnidirectionalLstmTest/.+ 255 256 # lstm_test 257 -LstmOpTest/InvalidTypes 258 # Float 259 Parameterized/LstmOpTest.+/0,29 260 Parameterized/LstmOpTest.+/1,29 261 Parameterized/LstmOpTest.+/2,29 262 Parameterized/LstmOpTest.+/3,29 263 # HybridUint8 264 Parameterized/LstmOpTest.+/4,29 265 Parameterized/LstmOpTest.+/5,29 266 Parameterized/LstmOpTest.+/6,29 267 Parameterized/LstmOpTest.+/7,29 268 # HybridInt8 269 -Parameterized/LstmOpTest.+/8 270 -Parameterized/LstmOpTest.+/9 271 -Parameterized/LstmOpTest.+/10 272 -Parameterized/LstmOpTest.+/11 273 274 # maximum_minimum_test 275 MaxMinOpTest/.+nt8Test,29 276 MaximumOpTest/.+,29 277 278 # mul_test 279 FloatMulOpTest/.+ 280 281 # neg_test 282 -NegOpModel/.+Int64 283 NegOpModel/.+,29 284 285 # pad_test 286 -PadOpTest/TooManyDimensions 287 -PadOpTest/UnequalDimensions 288 -PadOpTest/InvalidPadValue 289 # Zero height or width is not supported 290 -PadOpTest/Zero.+ConstImageStyleTest 291 # Dynamic tensors are not supported 292 -.*Pad.*OpTest/.+Dynamic.*Test 293 -QuantizedPad.*OpTest/.+ZeroNotInQuantizationRange 294 # 16-bit tests are not supported 295 -QuantizedPadOpTest/.*Int16.* 296 QuantizedPadOpTest/.+,29 297 QuantizedPadV2OpTest/.+,29 298 PadOpTest/.+,29 299 300 # pooling_test 301 FloatPoolingOpTest/L2PoolActivationRelu.*,29 302 FloatPoolingOpTest/.+ 303 # Image is too big 304 -QuantizedPoolingOpTest/AveragePoolImageSize17 305 # Int16 unsupported 306 -QuantizedPoolingOpTest/SymmetricAveragePool16 307 QuantizedPoolingOpTest/.+ 308 QuantizedUInt8PoolingOpTest/.+ 309 310 # pow_test 311 -PowOpModel/Simple 312 -PowOpModel/NegativeAndZeroValue 313 -PowOpModel/BroadcastTest 314 -PowOpModel/IntSingleIntegerExponentTest 315 PowOpModel/.+,29 316 317 # quant_basic_lstm_test 318 QuantizedLstmTest/BasicQuantizedLstmTest/29 319 320 # quantized_lstm op test 321 # Temporary disabled due to b/188515203 322 //IntegerLstmOpTest/NoCifg_NoPeephole_Projection_LayerNorm,30 323 324 # quantize_test 325 QuantizeOpTest/UINT8,29 326 QuantizeOpTest/UInt8UInt8.+,29 327 QuantizeOpTest/Int8Int8.+,30 328 QuantizeOpTest/INT8,30 329 330 # rank 331 332 # reduce_test 333 -Dynamic.+(Mean|Sum|Prod|Max|Min)OpTest/.+ 334 -ConstUint8(Mean|Sum)OpTest/.+ 335 -ConstInt8MeanOpTest.NonSpecialAxisNonSameScale 336 -ConstInt8MeanOpTest.QuantizedDifferentScale 337 ConstUint8(Max|Min)OpTest/.+,29 338 ConstUint8(Mean)OpTest/.+ 339 -ConstInt8(Max|Min)OpTest/.+,29 340 -ConstMeanOpTest.*/.+ 341 -MeanOpTestQuantized.*/.+ 342 ConstFloat(Sum|Prod|Max|Min)OpTest/NotKeepDims,29 343 ConstFloat(Sum|Prod|Max|Min)OpTest/KeepDims,29 344 ConstFloat(Mean|Any)OpTest/NotKeepDims 345 ConstFloat(Mean|Any)OpTest/KeepDims 346 ConstFloat(Sum|Prod|Max|Min)OpTest/ScalarAxis,29 347 348 # reshape_test 349 # Acceleration would be only for the test with shape being a constant tensor or 350 # as hardcoded options. 351 VariedShapeSpec/ReshapeOpTest/InvalidShape/[01] 352 VariedShapeSpec/ReshapeOpTest/RegularShapes/[01] 353 VariedShapeSpec/ReshapeOpTest/WithStretchDimension/[01] 354 355 # resize_bilinear_test 356 // align_corners & half_pixel_centers are not implemented in NNAPI before API 30 357 ResizeBilinearOpTest/ResizeBilinearOpTest.+HalfPixelCenters.*/0,30 358 // Only models with constant size tensor are accelerated 359 ResizeBilinearOpTest/ResizeBilinearOpTest/.+/0,29 360 361 # resize_nearest_neighbor_test 362 // align_corners & half_pixel_centers are not implemented in NNAPI before API 30 363 ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest.+AlignCorners.*/0,30 364 ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest.+HalfPixelCenters.*/0,30 365 // 16-bit tests are not supported 366 -ResizeNearestNeighborOpTest.+Int16/.+ 367 // Only models with constant size tensor are accelerated 368 ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest/.+/0,29 369 370 # select_test 371 -SelectOpTest/SelectBool 372 -SelectOpTest.SelectInt16 373 -SelectOpTest/RankZero.+ 374 -SelectOpTest/RankOne.+ 375 SelectOpTest/.+,29 376 377 # slice_test 378 -SliceOpTest/SliceOpTest/IndexInt64/.+ 379 -SliceOpTest/SliceOpTest/SliceString/.+ 380 -SliceOpTest/SliceOpTest/SliceInt64/.+ 381 -SliceOpTest/SliceOpTest/SliceBool/.+ 382 -SliceOpTest/SliceOpTest/SliceInt16/.+ 383 # Only constant tensors 384 SliceOpTest/SliceOpTest/.+/0,29 385 386 # softmax_test 387 SoftmaxOpTest/CompareWithTFminiBetaEq1 388 SoftmaxOpTest/CompareWithTFminiBetaNotEq1 389 390 # space_to_depth_test 391 SpaceToDepthOpModel/Float32 392 SpaceToDepthOpModel/Uint8 393 SpaceToDepthOpModel/int8 394 395 # split_test 396 -SplitOpTest/SplitOpTest/.+Int8/.+ 397 # Only accelerated when axis is a constant tensor 398 SplitOpTest/SplitOpTest/.+/0,29 399 400 # squeeze_test 401 FloatSqueezeOpTest/.+,29 402 403 # sub_test 404 -FloatSubOpModel/WithBroadcast5D 405 FloatSubOpModel/.+ 406 -QuantizedSubOpModel/.+Int16 407 -QuantizedSubOpModel/.+Int8 408 QuantizedSubOpModel/.+ 409 410 # svdf_test 411 SVDFOpTest/BlackBoxTestRank1 412 SVDFOpTest/BlackBoxTestRank2 413 414 # tile_test 415 -TileTest/TileTest/Int64.+/.+ 416 -TileTest/TileTest/Boolean.+/.+ 417 -TileTest/TileTest/String.+/.+ 418 # Const tensor only 419 TileTest/TileTest/.+/0,29 420 421 # topk_v2_test 422 -TopKV2OpTest/TopKV2OpTest/.+Int64/.+ 423 # Const tensor only 424 TopKV2OpTest/TopKV2OpTest/.+/0,29 425 426 # transpose_test 427 # death test 428 -TransposeTest/Test6DInputTensor 429 -TransposeTest/5DDividedIntoTwo2Ds.* 430 -TransposeTest/Complex5DTest.* 431 -TransposeTest/.+DynamicTensor 432 -TransposeTest/TestRefOps4DInt16 433 TransposeTest/.+ 434 435 # transpose_conv_test 436 -TransposeConvOpTest/TransposeConvOpTest.SimpleTestQuantizedPerChannel16x8/.+ 437 TransposeConvOpTest/TransposeConvOpTest..*Bias.*/0,29 438 # Const tensor only 439 TransposeConvOpTest/TransposeConvOpTest/.+/0,29 440 441 # unidirectional_sequence_rnn_test 442 UnidirectionalRNNOpTest/BlackBoxTest,29 443 UnidirectionalRNNOpTest.TimeMajorBlackBoxTest,29 444 )"; 445 446 } // namespace tflite 447