1#!/usr/bin/env python3 2# 3# [VPYTHON:BEGIN] 4# python_version: "3.8" 5# [VPYTHON:END] 6# 7# Copyright (C) 2021 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20 21import sys, os, argparse, subprocess, shlex, re, concurrent.futures, multiprocessing 22 23def parse_args(): 24 parser = argparse.ArgumentParser(description="Run libcore tests using the vogar testing tool.") 25 parser.add_argument('--mode', choices=['device', 'host', 'jvm'], required=True, 26 help='Specify where tests should be run.') 27 parser.add_argument('--variant', choices=['X32', 'X64'], 28 help='Which dalvikvm variant to execute with.') 29 parser.add_argument('-j', '--jobs', type=int, 30 help='Number of tests to run simultaneously.') 31 parser.add_argument('--timeout', type=int, 32 help='How long to run the test before aborting (seconds).') 33 parser.add_argument('--debug', action='store_true', 34 help='Use debug version of ART (device|host only).') 35 parser.add_argument('--dry-run', action='store_true', 36 help='Print vogar command-line, but do not run.') 37 parser.add_argument('--no-jit', action='store_false', dest='jit', 38 help='Disable JIT (device|host only).') 39 parser.add_argument('--gcstress', action='store_true', 40 help='Enable GC stress configuration (device|host only).') 41 parser.add_argument('tests', nargs="*", 42 help='Name(s) of the test(s) to run') 43 parser.add_argument('--verbose', action='store_true', help='Print verbose output from vogar.') 44 return parser.parse_args() 45 46ART_TEST_ANDROID_ROOT = os.environ.get("ART_TEST_ANDROID_ROOT", "/system") 47ART_TEST_CHROOT = os.environ.get("ART_TEST_CHROOT") 48ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT") 49 50LIBCORE_TEST_NAMES = [ 51 ### luni tests. ### 52 # Naive critical path optimization: Run the longest tests first. 53 "org.apache.harmony.tests.java.util", # 90min under gcstress 54 "libcore.java.lang", # 90min under gcstress 55 "jsr166", # 60min under gcstress 56 "libcore.java.util", # 60min under gcstress 57 "libcore.java.math", # 50min under gcstress 58 "org.apache.harmony.crypto", # 30min under gcstress 59 "org.apache.harmony.tests.java.io", # 30min under gcstress 60 "org.apache.harmony.tests.java.text", # 30min under gcstress 61 # Split highmemorytest to individual classes since it is too big. 62 "libcore.highmemorytest.java.text.DateFormatTest", 63 "libcore.highmemorytest.java.text.DecimalFormatTest", 64 "libcore.highmemorytest.java.text.SimpleDateFormatTest", 65 "libcore.highmemorytest.java.time.format.DateTimeFormatterTest", 66 "libcore.highmemorytest.java.util.CalendarTest", 67 "libcore.highmemorytest.java.util.CurrencyTest", 68 "libcore.highmemorytest.libcore.icu.SimpleDateFormatDataTest", 69 # All other luni tests in alphabetical order. 70 "libcore.android.system", 71 "libcore.build", 72 "libcore.dalvik.system", 73 "libcore.java.awt", 74 "libcore.java.text", 75 "libcore.javax.crypto", 76 "libcore.javax.net", 77 "libcore.javax.security", 78 "libcore.javax.sql", 79 "libcore.javax.xml", 80 "libcore.libcore.icu", 81 "libcore.libcore.internal", 82 "libcore.libcore.io", 83 "libcore.libcore.net", 84 "libcore.libcore.reflect", 85 "libcore.libcore.util", 86 "libcore.sun.invoke", 87 "libcore.sun.misc", 88 "libcore.sun.net", 89 "libcore.sun.security", 90 "libcore.sun.util", 91 "libcore.xml", 92 "org.apache.harmony.annotation", 93 "org.apache.harmony.luni.tests.internal.net.www.protocol.http.HttpURLConnection", 94 "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnection", 95 "org.apache.harmony.luni.tests.java.io", 96 "org.apache.harmony.luni.tests.java.net", 97 "org.apache.harmony.nio", 98 "org.apache.harmony.regex", 99 "org.apache.harmony.testframework", 100 "org.apache.harmony.tests.java.lang", 101 "org.apache.harmony.tests.java.math", 102 "org.apache.harmony.tests.javax.security", 103 "tests.java.lang.String", 104 ### OpenJDK upstream tests (ojluni). ### 105 # "test.java.awt", 106 "test.java.awt", 107 # test.java.io 108 "test.java.io.ByteArrayInputStream", 109 "test.java.io.ByteArrayOutputStream", 110 "test.java.io.FileReader", 111 "test.java.io.FileWriter", 112 "test.java.io.InputStream", 113 "test.java.io.OutputStream", 114 "test.java.io.PrintStream", 115 "test.java.io.PrintWriter", 116 "test.java.io.Reader", 117 "test.java.io.Writer", 118 # test.java.lang 119 "test.java.lang.Boolean", 120 "test.java.lang.ClassLoader", 121 "test.java.lang.Double", 122 "test.java.lang.Float", 123 "test.java.lang.Integer", 124 "test.java.lang.Long", 125 # Sharded test.java.lang.StrictMath 126 "test.java.lang.StrictMath.CubeRootTests", 127 # TODO: disable the test until b/248208762 is fixed. 128 # "test.java.lang.StrictMath.ExactArithTests", 129 "test.java.lang.StrictMath.Expm1Tests", 130 "test.java.lang.StrictMath.ExpTests", 131 "test.java.lang.StrictMath.HyperbolicTests", 132 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard1", 133 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard2", 134 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard3", 135 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard4", 136 "test.java.lang.StrictMath.HypotTests#testHypot", 137 "test.java.lang.StrictMath.Log1pTests", 138 "test.java.lang.StrictMath.Log10Tests", 139 "test.java.lang.StrictMath.MultiplicationTests", 140 "test.java.lang.StrictMath.PowTests", 141 "test.java.lang.String", 142 "test.java.lang.Thread", 143 # test.java.lang.invoke 144 "test.java.lang.invoke", 145 # test.java.lang.ref 146 "test.java.lang.ref.SoftReference", 147 "test.java.lang.ref.BasicTest", 148 "test.java.lang.ref.EnqueueNullRefTest", 149 "test.java.lang.ref.EnqueuePollRaceTest", 150 "test.java.lang.ref.ReferenceCloneTest", 151 "test.java.lang.ref.ReferenceEnqueuePendingTest", 152 # test.java.lang.runtime 153 "test.java.lang.runtime.SwitchBootstrapsTest", 154 # test.java.math 155 "test.java.math.BigDecimal", 156 # Sharded test.java.math.BigInteger 157 "test.java.math.BigInteger#testArithmetic", 158 "test.java.math.BigInteger#testBitCount", 159 "test.java.math.BigInteger#testBitLength", 160 "test.java.math.BigInteger#testbitOps", 161 "test.java.math.BigInteger#testBitwise", 162 "test.java.math.BigInteger#testByteArrayConv", 163 "test.java.math.BigInteger#testConstructor", 164 "test.java.math.BigInteger#testDivideAndReminder", 165 "test.java.math.BigInteger#testDivideLarge", 166 "test.java.math.BigInteger#testModExp", 167 "test.java.math.BigInteger#testMultiplyLarge", 168 "test.java.math.BigInteger#testNextProbablePrime", 169 "test.java.math.BigInteger#testPow", 170 "test.java.math.BigInteger#testSerialize", 171 "test.java.math.BigInteger#testShift", 172 "test.java.math.BigInteger#testSquare", 173 "test.java.math.BigInteger#testSquareLarge", 174 "test.java.math.BigInteger#testSquareRootAndReminder", 175 "test.java.math.BigInteger#testStringConv_generic", 176 "test.java.math.RoundingMode", 177 # test.java.net 178 "test.java.net.DatagramSocket", 179 "test.java.net.Socket", 180 "test.java.net.SocketOptions", 181 "test.java.net.URLDecoder", 182 "test.java.net.URLEncoder", 183 # test.java.nio 184 "test.java.nio.channels.Channels", 185 "test.java.nio.channels.SelectionKey", 186 "test.java.nio.channels.Selector", 187 "test.java.nio.file", 188 # test.java.security 189 "test.java.security.cert", 190 # Sharded test.java.security.KeyAgreement 191 "test.java.security.KeyAgreement.KeyAgreementTest", 192 "test.java.security.KeyAgreement.KeySizeTest#testECDHKeySize", 193 "test.java.security.KeyAgreement.KeySpecTest", 194 "test.java.security.KeyAgreement.MultiThreadTest", 195 "test.java.security.KeyAgreement.NegativeTest", 196 "test.java.security.KeyStore", 197 "test.java.security.Provider", 198 # test.java.time 199 "test.java.time", 200 # test.java.util 201 "test.java.util.Arrays", 202 "test.java.util.Collection", 203 "test.java.util.Collections", 204 "test.java.util.Date", 205 "test.java.util.EnumMap", 206 "test.java.util.EnumSet", 207 "test.java.util.GregorianCalendar", 208 "test.java.util.LinkedHashMap", 209 "test.java.util.LinkedHashSet", 210 "test.java.util.List", 211 "test.java.util.Map", 212 "test.java.util.Optional", 213 "test.java.util.TestFormatter", 214 "test.java.util.TimeZone", 215 # test.java.util.concurrent 216 "test.java.util.concurrent", 217 # test.java.util.function 218 "test.java.util.function", 219 # test.java.util.stream 220 "test.java.util.stream", 221 # test.java.util.zip 222 "test.java.util.zip.ZipFile", 223 # tck.java.time 224 "tck.java.time", 225] 226# "org.apache.harmony.security", # We don't have rights to revert changes in case of failures. 227 228# Note: This must start with the CORE_IMG_JARS in Android.common_path.mk 229# because that's what we use for compiling the boot.art image. 230# It may contain additional modules from TEST_CORE_JARS. 231BOOT_CLASSPATH = [ 232 "/apex/com.android.art/javalib/core-oj.jar", 233 "/apex/com.android.art/javalib/core-libart.jar", 234 "/apex/com.android.art/javalib/okhttp.jar", 235 "/apex/com.android.art/javalib/bouncycastle.jar", 236 "/apex/com.android.art/javalib/apache-xml.jar", 237 "/apex/com.android.i18n/javalib/core-icu4j.jar", 238 "/apex/com.android.conscrypt/javalib/conscrypt.jar", 239] 240 241CLASSPATH = ["core-tests", "core-ojtests", "jsr166-tests", "mockito-target"] 242 243SLOW_OJLUNI_TESTS = { 244 "test.java.awt", 245 "test.java.lang.String", 246 "test.java.lang.invoke", 247 "test.java.nio.channels.Selector", 248 "test.java.time", 249 "test.java.util.Arrays", 250 "test.java.util.Map", 251 "test.java.util.concurrent", 252 "test.java.util.stream", 253 "test.java.util.zip.ZipFile", 254 "tck.java.time", 255} 256 257# Disabled to unblock art-buildbot 258# These tests fail with "java.io.IOException: Stream closed", tracked in 259# http://b/235566533 and http://b/208639267 260DISABLED_GCSTRESS_DEBUG_TESTS = { 261 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard1", 262 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard2", 263 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard3", 264 "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard4", 265 "test.java.math.BigDecimal", 266 "test.java.math.BigInteger#testConstructor", 267 "test.java.util.TestFormatter", 268 "test.java.util.Collection", 269} 270 271def get_jar_filename(classpath): 272 base_path = (ANDROID_PRODUCT_OUT + "/../..") if ANDROID_PRODUCT_OUT else "out/target" 273 base_path = os.path.normpath(base_path) # Normalize ".." components for readability. 274 return f"{base_path}/common/obj/JAVA_LIBRARIES/{classpath}_intermediates/classes.jar" 275 276def get_timeout_secs(): 277 default_timeout_secs = 600 278 if args.gcstress: 279 default_timeout_secs = 1200 280 if args.debug: 281 default_timeout_secs = 1800 282 return args.timeout or default_timeout_secs 283 284def get_expected_failures(): 285 failures = ["art/tools/libcore_failures.txt"] 286 if args.mode != "jvm": 287 if args.gcstress: 288 failures.append("art/tools/libcore_gcstress_failures.txt") 289 if args.gcstress and args.debug: 290 failures.append("art/tools/libcore_gcstress_debug_failures.txt") 291 if args.debug and not args.gcstress: 292 failures.append("art/tools/libcore_debug_failures.txt") 293 return failures 294 295def get_test_names(): 296 if args.tests: 297 return args.tests 298 test_names = list(LIBCORE_TEST_NAMES) 299 # See b/78228743 and b/178351808. 300 if args.gcstress or args.debug or args.mode == "jvm": 301 test_names = list(t for t in test_names if not t.startswith("libcore.highmemorytest")) 302 test_names = list(filter(lambda x: x not in SLOW_OJLUNI_TESTS, test_names)) 303 if args.gcstress and args.debug: 304 test_names = list(filter(lambda x: x not in DISABLED_GCSTRESS_DEBUG_TESTS, test_names)) 305 return test_names 306 307def get_vogar_command(test_name): 308 cmd = ["vogar"] 309 if args.mode == "device": 310 cmd.append("--mode=device --vm-arg -Ximage:/system/framework/art_boot_images/boot.art") 311 cmd.append("--vm-arg -Xbootclasspath:" + ":".join(BOOT_CLASSPATH)) 312 313 if args.mode == "host": 314 # We explicitly give a wrong path for the image, to ensure vogar 315 # will create a boot image with the default compiler. Note that 316 # giving an existing image on host does not work because of 317 # classpath/resources differences when compiling the boot image. 318 cmd.append("--mode=host --vm-arg -Ximage:/non/existent/vogar.art") 319 if args.mode == "jvm": 320 cmd.append("--mode=jvm") 321 if args.variant: 322 cmd.append("--variant=" + args.variant) 323 if args.gcstress: 324 cmd.append("--vm-arg -Xgc:gcstress") 325 cmd.append('--vm-arg -Djsr166.delay.factor="1.50"') 326 if args.debug: 327 cmd.append("--vm-arg -XXlib:libartd.so --vm-arg -XX:SlowDebug=true") 328 329 if args.mode == "device": 330 if ART_TEST_CHROOT: 331 cmd.append(f"--chroot {ART_TEST_CHROOT} --device-dir=/tmp/vogar/test-{test_name}") 332 else: 333 cmd.append(f"--device-dir=/data/local/tmp/vogar/test-{test_name}") 334 cmd.append(f"--vm-command={ART_TEST_ANDROID_ROOT}/bin/art") 335 else: 336 cmd.append(f"--device-dir=/tmp/vogar/test-{test_name}") 337 338 if args.mode != "jvm": 339 cmd.append("--timeout {}".format(get_timeout_secs())) 340 cmd.append("--toolchain d8 --language CUR") 341 if args.jit: 342 cmd.append("--vm-arg -Xcompiler-option --vm-arg --compiler-filter=verify") 343 cmd.append("--vm-arg -Xusejit:{}".format(str(args.jit).lower())) 344 345 if args.verbose: 346 cmd.append("--verbose") 347 348 # Suppress color codes if not attached to a terminal 349 if not sys.stdout.isatty(): 350 cmd.append("--no-color") 351 352 cmd.extend("--expectations " + f for f in get_expected_failures()) 353 cmd.extend("--classpath " + get_jar_filename(cp) for cp in CLASSPATH) 354 cmd.append(test_name) 355 356 # vogar target options 357 if not os.path.exists('frameworks/base'): 358 cmd.append("--") 359 # Skip @NonMts test in thin manifest which uses prebuilt Conscrypt and ICU. 360 # It's similar to running libcore tests on the older platforms. 361 # @NonMts means that the test doesn't pass on a older platform version. 362 cmd.append("--exclude-filter libcore.test.annotation.NonMts") 363 return cmd 364 365def get_target_cpu_count(): 366 adb_command = 'adb shell cat /sys/devices/system/cpu/present' 367 with subprocess.Popen(adb_command.split(), 368 stderr=subprocess.STDOUT, 369 stdout=subprocess.PIPE, 370 universal_newlines=True) as proc: 371 assert(proc.wait() == 0) # Check the exit code. 372 match = re.match(r'\d*-(\d*)', proc.stdout.read()) 373 assert(match) 374 return int(match.group(1)) + 1 # Add one to convert from "last-index" to "count" 375 376def main(): 377 global args 378 args = parse_args() 379 380 if not os.path.exists('build/envsetup.sh'): 381 raise AssertionError("Script needs to be run at the root of the android tree") 382 for jar in map(get_jar_filename, CLASSPATH): 383 if not os.path.exists(jar): 384 raise AssertionError(f"Missing {jar}. Run buildbot-build.sh first.") 385 386 if not args.jobs: 387 if args.mode == "device": 388 args.jobs = get_target_cpu_count() 389 else: 390 args.jobs = multiprocessing.cpu_count() 391 if args.gcstress: 392 # TODO: Investigate and fix the underlying issues. 393 args.jobs = args.jobs // 2 394 395 def run_test(test_name): 396 cmd = " ".join(get_vogar_command(test_name)) 397 if args.dry_run: 398 return test_name, cmd, "Dry-run: skipping execution", 0 399 with subprocess.Popen(shlex.split(cmd), 400 stderr=subprocess.STDOUT, 401 stdout=subprocess.PIPE, 402 universal_newlines=True) as proc: 403 return test_name, cmd, proc.communicate()[0], proc.wait() 404 405 failed_regex = re.compile(r"^.* FAIL \((?:EXEC_FAILED|ERROR)\)$", re.MULTILINE) 406 failed_tests, max_exit_code = [], 0 407 with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool: 408 futures = [pool.submit(run_test, test_name) for test_name in get_test_names()] 409 print(f"Running {len(futures)} tasks on {args.jobs} core(s)...\n") 410 for i, future in enumerate(concurrent.futures.as_completed(futures)): 411 test_name, cmd, stdout, exit_code = future.result() 412 if exit_code != 0 or args.dry_run or args.verbose: 413 print(cmd) 414 print(stdout.strip()) 415 else: 416 print(stdout.strip().split("\n")[-1]) # Vogar final summary line. 417 failed_match = failed_regex.findall(stdout) 418 failed_tests.extend(failed_match) 419 max_exit_code = max(max_exit_code, exit_code) 420 result = "PASSED" if exit_code == 0 else f"FAILED ({len(failed_match)} test(s) failed)" 421 print(f"[{i+1}/{len(futures)}] Test set {test_name} {result}\n") 422 print(f"Overall, {len(failed_tests)} test(s) failed:") 423 print("\n".join(failed_tests)) 424 sys.exit(max_exit_code) 425 426if __name__ == '__main__': 427 main() 428