1#!/usr/bin/env python 2# Copyright 2013 The Flutter Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" 7A top level harness to run all unit-tests in a specific engine build. 8""" 9 10import sys 11import os 12import argparse 13import glob 14import subprocess 15 16buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..')) 17out_dir = os.path.join(buildroot_dir, 'out') 18golden_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'resources') 19fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts') 20roboto_font_path = os.path.join(fonts_dir, 'Roboto-Regular.ttf') 21dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',) 22 23fonts_dir_flag = '--font-directory=%s' % fonts_dir 24time_sensitve_test_flag = '--gtest_filter="-*TimeSensitiveTest*"' 25 26def IsMac(): 27 return sys.platform == 'darwin' 28 29 30def IsLinux(): 31 return sys.platform.startswith('linux') 32 33 34def IsWindows(): 35 return sys.platform.startswith(('cygwin', 'win')) 36 37 38def ExecutableSuffix(): 39 return '.exe' if IsWindows() else '' 40 41def FindExecutablePath(path): 42 if os.path.exists(path): 43 return path 44 45 if IsWindows(): 46 exe_path = path + '.exe' 47 if os.path.exists(exe_path): 48 return exe_path 49 50 bat_path = path + '.bat' 51 if os.path.exists(bat_path): 52 return bat_path 53 54 raise Exception('Executable %s does not exist!' % path) 55 56 57def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir): 58 if filter is not None and executable_name not in filter: 59 print 'Skipping %s due to filter.' % executable_name 60 return 61 62 executable = FindExecutablePath(os.path.join(build_dir, executable_name)) 63 64 print 'Running %s in %s' % (executable_name, cwd) 65 test_command = [ executable ] + flags 66 print ' '.join(test_command) 67 subprocess.check_call(test_command, cwd=cwd) 68 69 70def RunCCTests(build_dir, filter): 71 print "Running Engine Unit-tests." 72 73 RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter) 74 75 RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter) 76 77 # https://github.com/flutter/flutter/issues/36294 78 if not IsWindows(): 79 RunEngineExecutable(build_dir, 'embedder_unittests', filter) 80 81 flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold'] 82 if IsLinux(): 83 flow_flags = [ 84 '--golden-dir=%s' % golden_dir, 85 '--font-file=%s' % roboto_font_path, 86 ] 87 RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags) 88 89 RunEngineExecutable(build_dir, 'fml_unittests', filter, [ time_sensitve_test_flag ]) 90 91 RunEngineExecutable(build_dir, 'runtime_unittests', filter) 92 93 # https://github.com/flutter/flutter/issues/36295 94 if not IsWindows(): 95 RunEngineExecutable(build_dir, 'shell_unittests', filter) 96 97 RunEngineExecutable(build_dir, 'ui_unittests', filter) 98 99 # These unit-tests are Objective-C and can only run on Darwin. 100 if IsMac(): 101 RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter) 102 103 # https://github.com/flutter/flutter/issues/36296 104 if IsLinux(): 105 RunEngineExecutable(build_dir, 'txt_unittests', filter, [ fonts_dir_flag ]) 106 107 108def RunEngineBenchmarks(build_dir, filter): 109 print "Running Engine Benchmarks." 110 111 RunEngineExecutable(build_dir, 'shell_benchmarks', filter) 112 113 RunEngineExecutable(build_dir, 'fml_benchmarks', filter) 114 115 if IsLinux(): 116 RunEngineExecutable(build_dir, 'txt_benchmarks', filter, [ fonts_dir_flag ]) 117 118 119 120def SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot): 121 print "Generating snapshot for test %s" % dart_file 122 123 dart = os.path.join(build_dir, 'dart-sdk', 'bin', 'dart') 124 frontend_server = os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot') 125 flutter_patched_sdk = os.path.join(build_dir, 'flutter_patched_sdk') 126 test_packages = os.path.join(dart_tests_dir, '.packages') 127 128 assert os.path.exists(dart) 129 assert os.path.exists(frontend_server) 130 assert os.path.exists(flutter_patched_sdk) 131 assert os.path.exists(test_packages) 132 133 snapshot_command = [ 134 dart, 135 frontend_server, 136 '--sdk-root', 137 flutter_patched_sdk, 138 '--incremental', 139 '--strong', 140 '--target=flutter', 141 '--packages', 142 test_packages, 143 '--output-dill', 144 kernel_file_output, 145 dart_file 146 ] 147 148 if verbose_dart_snapshot: 149 subprocess.check_call(snapshot_command, cwd=buildroot_dir) 150 else: 151 with open(os.devnull,"w") as out_file: 152 subprocess.check_call(snapshot_command, cwd=buildroot_dir, stdout=out_file) 153 assert os.path.exists(kernel_file_output) 154 155 156def RunDartTest(build_dir, dart_file, verbose_dart_snapshot): 157 kernel_file_name = os.path.basename(dart_file) + '.kernel.dill' 158 kernel_file_output = os.path.join(out_dir, kernel_file_name) 159 160 SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot) 161 162 print "Running test '%s' using 'flutter_tester'" % kernel_file_name 163 RunEngineExecutable(build_dir, 'flutter_tester', None, [ 164 '--disable-observatory', 165 '--use-test-fonts', 166 kernel_file_output 167 ]) 168 169def RunPubGet(build_dir, directory): 170 print "Running 'pub get' in the tests directory %s" % dart_tests_dir 171 172 pub_get_command = [ 173 os.path.join(build_dir, 'dart-sdk', 'bin', 'pub'), 174 'get' 175 ] 176 subprocess.check_call(pub_get_command, cwd=directory) 177 178 179def EnsureDebugUnoptSkyPackagesAreBuilt(): 180 variant_out_dir = os.path.join(out_dir, 'host_debug_unopt') 181 182 ninja_command = [ 183 'ninja', 184 '-C', 185 variant_out_dir, 186 'flutter/sky/packages' 187 ] 188 189 # Attempt running Ninja if the out directory exists. 190 # We don't want to blow away any custom GN args the caller may have already set. 191 if os.path.exists(variant_out_dir): 192 subprocess.check_call(ninja_command, cwd=buildroot_dir) 193 return 194 195 gn_command = [ 196 os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'), 197 '--runtime-mode', 198 'debug', 199 '--unopt', 200 '--no-lto', 201 ] 202 203 subprocess.check_call(gn_command, cwd=buildroot_dir) 204 subprocess.check_call(ninja_command, cwd=buildroot_dir) 205 206def EnsureJavaTestsAreBuilt(android_out_dir): 207 ninja_command = [ 208 'ninja', 209 '-C', 210 android_out_dir, 211 'flutter/shell/platform/android:robolectric_tests' 212 ] 213 214 # Attempt running Ninja if the out directory exists. 215 # We don't want to blow away any custom GN args the caller may have already set. 216 if os.path.exists(android_out_dir): 217 subprocess.check_call(ninja_command, cwd=buildroot_dir) 218 return 219 220 # Otherwise prepare the directory first, then build the test. 221 gn_command = [ 222 os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'), 223 '--android', 224 '--unoptimized', 225 '--runtime-mode=debug', 226 '--no-lto', 227 ] 228 subprocess.check_call(gn_command, cwd=buildroot_dir) 229 subprocess.check_call(ninja_command, cwd=buildroot_dir) 230 231def RunJavaTests(filter, android_variant='android_debug_unopt'): 232 android_out_dir = os.path.join(out_dir, android_variant) 233 EnsureJavaTestsAreBuilt(android_out_dir) 234 235 robolectric_dir = os.path.join(buildroot_dir, 'third_party', 'robolectric', 'lib') 236 classpath = map(str, [ 237 os.path.join(buildroot_dir, 'third_party', 'android_tools', 'sdk', 'platforms', 'android-29', 'android.jar'), 238 os.path.join(robolectric_dir, '*'), # Wildcard for all jars in the directory 239 os.path.join(android_out_dir, 'flutter.jar'), 240 os.path.join(android_out_dir, 'robolectric_tests.jar') 241 ]) 242 243 test_class = filter if filter else 'io.flutter.FlutterTestSuite' 244 command = [ 245 'java', 246 '-Drobolectric.offline=true', 247 '-Drobolectric.dependency.dir=' + robolectric_dir, 248 '-classpath', ':'.join(classpath), 249 '-Drobolectric.logging=stdout', 250 'org.junit.runner.JUnitCore', 251 test_class 252 ] 253 254 return subprocess.check_call(command) 255 256def RunDartTests(build_dir, filter, verbose_dart_snapshot): 257 # This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml 258 # has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/ 259 # Before running Dart tests, make sure to run just that target (NOT the whole engine) 260 EnsureDebugUnoptSkyPackagesAreBuilt(); 261 262 # Now that we have the Sky packages at the hardcoded location, run `pub get`. 263 RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), None, flags=['get'], cwd=dart_tests_dir) 264 265 dart_tests = glob.glob('%s/*.dart' % dart_tests_dir) 266 267 for dart_test_file in dart_tests: 268 if filter is not None and os.path.basename(dart_test_file) not in filter: 269 print "Skipping %s due to filter." % dart_test_file 270 else: 271 print "Testing dart file %s" % dart_test_file 272 RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot) 273 274def main(): 275 parser = argparse.ArgumentParser(); 276 277 parser.add_argument('--variant', dest='variant', action='store', 278 default='host_debug_unopt', help='The engine build variant to run the tests for.'); 279 parser.add_argument('--type', type=str, default='all') 280 parser.add_argument('--engine-filter', type=str, default='', 281 help='A list of engine test executables to run.') 282 parser.add_argument('--dart-filter', type=str, default='', 283 help='A list of Dart test scripts to run.') 284 parser.add_argument('--java-filter', type=str, default='', 285 help='A single Java test class to run.') 286 parser.add_argument('--android-variant', dest='android_variant', action='store', 287 default='android_debug_unopt', 288 help='The engine build variant to run java tests for') 289 parser.add_argument('--verbose-dart-snapshot', dest='verbose_dart_snapshot', action='store_true', 290 default=False, help='Show extra dart snapshot logging.') 291 292 args = parser.parse_args() 293 294 if args.type == 'all': 295 types = ['engine', 'dart', 'benchmarks', 'java'] 296 else: 297 types = args.type.split(',') 298 299 build_dir = os.path.join(out_dir, args.variant) 300 if args.type != 'java': 301 assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir 302 303 engine_filter = args.engine_filter.split(',') if args.engine_filter else None 304 if 'engine' in types: 305 RunCCTests(build_dir, engine_filter) 306 307 if 'dart' in types: 308 assert not IsWindows(), "Dart tests can't be run on windows. https://github.com/flutter/flutter/issues/36301." 309 dart_filter = args.dart_filter.split(',') if args.dart_filter else None 310 RunDartTests(build_dir, dart_filter, args.verbose_dart_snapshot) 311 312 if 'java' in types: 313 assert not IsWindows(), "Android engine files can't be compiled on Windows." 314 java_filter = args.java_filter 315 if ',' in java_filter or '*' in java_filter: 316 print('Can only filter JUnit4 tests by single entire class name, eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter) 317 java_filter = None 318 RunJavaTests(java_filter, args.android_variant) 319 320 # https://github.com/flutter/flutter/issues/36300 321 if 'benchmarks' in types and not IsWindows(): 322 RunEngineBenchmarks(build_dir, engine_filter) 323 324 325if __name__ == '__main__': 326 sys.exit(main()) 327