1#! /usr/bin/env python3 2# 3# Copyright 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17# Regenerate some ART test related files. 18 19# This script handles only a subset of ART run-tests at the moment; additional 20# cases will be added later. 21 22import argparse 23import copy 24import collections 25import itertools 26import json 27import logging 28import os 29import re 30import sys 31import textwrap 32import xml.dom.minidom 33 34logging.basicConfig(format='%(levelname)s: %(message)s') 35 36ME = os.path.basename(sys.argv[0]) 37 38# Common advisory placed at the top of all generated files. 39ADVISORY = f"Generated by `{ME}`. Do not edit manually." 40 41# Default indentation unit. 42INDENT = " " 43 44# Indentation unit for XML files. 45XML_INDENT = " " 46 47def reindent(str, indent = ""): 48 """Reindent literal string while removing common leading spaces.""" 49 return textwrap.indent(textwrap.dedent(str), indent) 50 51def copyright_header_text(year): 52 """Return the copyright header text used in XML files.""" 53 return reindent(f"""\ 54 Copyright (C) {year} The Android Open Source Project 55 56 Licensed under the Apache License, Version 2.0 (the "License"); 57 you may not use this file except in compliance with the License. 58 You may obtain a copy of the License at 59 60 http://www.apache.org/licenses/LICENSE-2.0 61 62 Unless required by applicable law or agreed to in writing, software 63 distributed under the License is distributed on an "AS IS" BASIS, 64 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 65 See the License for the specific language governing permissions and 66 limitations under the License. 67 """, " ") 68 69def split_list(l, n): 70 """Return a list of `n` sublists of (contiguous) elements of list `l`.""" 71 assert n > 0 72 (d, m) = divmod(len(l), n) 73 # If the length of `l` is divisible by `n`, use that that divisor (`d`) as size of each sublist; 74 # otherwise, the next integer value (`d + 1`). 75 s = d if m == 0 else d + 1 76 result = [l[i:i + s] for i in range(0, len(l), s)] 77 assert len(result) == n 78 return result 79 80# The prefix used in the Soong module name of all ART run-tests. 81ART_RUN_TEST_MODULE_NAME_PREFIX = "art-run-test-" 82 83# Number of shards used to declare ART run-tests in the sharded ART MTS test plan. 84NUM_MTS_ART_RUN_TEST_SHARDS = 1 85 86# Curated list of tests that have a custom `run` script, but that are 87# known to work fine with the default test execution strategy (i.e. 88# when ignoring their `run` script), even if not exactly as they would 89# with the original ART run-test harness. 90runnable_test_exceptions = frozenset([ 91 "055-enum-performance", 92 "059-finalizer-throw", 93 "080-oom-throw", 94 "1004-checker-volatile-ref-load", 95 "133-static-invoke-super", 96 "1338-gc-no-los", 97 "159-app-image-fields", 98 "160-read-barrier-stress", 99 "163-app-image-methods", 100 "165-lock-owner-proxy", 101 "168-vmstack-annotated", 102 "176-app-image-string", 103 "304-method-tracing", 104 "628-vdex", 105 "643-checker-bogus-ic", 106 "676-proxy-jit-at-first-use", 107 "677-fsi2", 108 "678-quickening", 109 "818-clinit-nterp", 110 "821-madvise-willneed", 111]) 112 113known_slow_tests = frozenset([ 114 "175-alloc-big-bignums", 115]) 116 117# Known failing ART run-tests. 118# TODO(rpl): Investigate and address the causes of failures. 119known_failing_tests = frozenset([ 120 "004-SignalTest", 121 "004-UnsafeTest", 122 "051-thread", 123 "086-null-super", 124 "087-gc-after-link", 125 # 1002-notify-startup: Dependency on `libarttest` + custom `check` script. 126 "1002-notify-startup", 127 "1337-gc-coverage", 128 "1339-dead-reference-safe", 129 "136-daemon-jni-shutdown", 130 "139-register-natives", 131 "148-multithread-gc-annotations", 132 "149-suspend-all-stress", 133 "150-loadlibrary", 134 "154-gc-loop", 135 "169-threadgroup-jni", 136 "177-visibly-initialized-deadlock", 137 "179-nonvirtual-jni", 138 "1945-proxy-method-arguments", 139 "2011-stack-walk-concurrent-instrument", 140 # 2040-huge-native-alloc: Fails with: 141 # 142 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 143 # at Main.main(Main.java:56) 144 # 145 "2040-huge-native-alloc", 146 "203-multi-checkpoint", 147 "2033-shutdown-mechanics", 148 "2036-jni-filechannel", 149 "2037-thread-name-inherit", 150 "2235-JdkUnsafeTest", 151 "305-other-fault-handler", 152 # 449-checker-bce: Dependency on `libarttest`. 153 "449-checker-bce", 154 "454-get-vreg", 155 "461-get-reference-vreg", 156 "466-get-live-vreg", 157 "497-inlining-and-class-loader", 158 "530-regression-lse", 159 "555-UnsafeGetLong-regression", 160 # 596-monitor-inflation: Dependency on `libarttest`. 161 "596-monitor-inflation", 162 "602-deoptimizeable", 163 "604-hot-static-interface", 164 "616-cha-native", 165 "616-cha-regression-proxy-method", 166 # 623-checker-loop-regressions: Dependency on `libarttest`. 167 "623-checker-loop-regressions", 168 "626-set-resolved-string", 169 "642-fp-callees", 170 "647-jni-get-field-id", 171 "655-jit-clinit", 172 "656-loop-deopt", 173 "664-aget-verifier", 174 # 680-checker-deopt-dex-pc-0: Dependency on `libarttest`. 175 "680-checker-deopt-dex-pc-0", 176 "685-deoptimizeable", 177 "687-deopt", 178 "693-vdex-inmem-loader-evict", 179 "708-jit-cache-churn", 180 # 716-jli-jit-samples: Dependency on `libarttest`. 181 "716-jli-jit-samples", 182 "717-integer-value-of", 183 "720-thread-priority", 184 # 730-cha-deopt: Fails with: 185 # 186 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 187 # at Main.main(Main.java:24) 188 # 189 "730-cha-deopt", 190 # 813-fp-args: Dependency on `libarttest`. 191 "813-fp-args", 192 # 821-many-args: Fails with: 193 # 194 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 195 # at Main.main(Main.java:20) 196 # 197 "821-many-args", 198 # 823-cha-inlining: Dependency on `libarttest`. 199 "823-cha-inlining", 200 # 826-infinite-loop: The test expects an argument passed to `Main.main` (the test library, 201 # usually `arttestd` or `arttest)`, but the ART run-test TradeFed test runner 202 # (`com.android.tradefed.testtype.ArtRunTest`) does not implement this yet. 203 "826-infinite-loop", 204 # 832-cha-recursive: Dependency on `libarttest`. 205 "832-cha-recursive", 206 # 837-deopt: Dependency on `libarttest`. 207 "837-deopt", 208 # 844-exception: Dependency on `libarttest`. 209 "844-exception", 210 # 844-exception2: Dependency on `libarttest`. 211 "844-exception2", 212 # 966-default-conflict: Dependency on `libarttest`. 213 "966-default-conflict", 214 # These tests need native code. 215 "993-breakpoints-non-debuggable", 216 "2243-single-step-default", 217 "2262-miranda-methods", 218 "2262-default-conflict-methods" 219]) 220 221known_failing_on_hwasan_tests = frozenset([ 222 "BootImageProfileTest", # b/232012605 223 "CtsJdwpTestCases", # times out 224]) 225 226# ART gtests that do not need root access to the device. 227art_gtest_user_module_names = [ 228 "art_libnativebridge_cts_tests", 229 "art_standalone_artd_tests", 230 "art_standalone_cmdline_tests", 231 "art_standalone_compiler_tests", 232 "art_standalone_dex2oat_tests", 233 "art_standalone_dexdump_tests", 234 "art_standalone_dexlist_tests", 235 "art_standalone_libartbase_tests", 236 "art_standalone_libartpalette_tests", 237 "art_standalone_libartservice_tests", 238 "art_standalone_libarttools_tests", 239 "art_standalone_libdexfile_support_tests", 240 "art_standalone_libdexfile_tests", 241 "art_standalone_libprofile_tests", 242 "art_standalone_oatdump_tests", 243 "art_standalone_odrefresh_tests", 244 "art_standalone_runtime_tests", 245 "art_standalone_sigchain_tests", 246 "libnativeloader_test", 247] 248 249# ART gtests that need root access to the device. 250art_gtest_eng_only_module_names = [ 251 "art_standalone_dexoptanalyzer_tests", 252 "art_standalone_profman_tests", 253 "libnativeloader_e2e_tests", 254] 255 256# All supported ART gtests. 257art_gtest_module_names = sorted(art_gtest_user_module_names + art_gtest_eng_only_module_names) 258 259# ART gtests supported in MTS that do not need root access to the device. 260art_gtest_mts_user_module_names = copy.copy(art_gtest_user_module_names) 261 262# ART gtests supported in Mainline presubmits. 263art_gtests_mainline_presubmit_module_names = copy.copy(art_gtest_module_names) 264 265# Tests exhibiting a flaky behavior, currently exluded from MTS for 266# the stake of stability / confidence (b/209958457). 267flaky_tests_excluded_from_mts = { 268 "CtsLibcoreFileIOTestCases": [ 269 ("android.cts.FileChannelInterProcessLockTest#" + m) for m in [ 270 "test_lockJJZ_Exclusive_asyncChannel", 271 "test_lockJJZ_Exclusive_syncChannel", 272 "test_lock_differentChannelTypes", 273 "test_lockJJZ_Shared_asyncChannel", 274 "test_lockJJZ_Shared_syncChannel", 275 ] 276 ], 277 "CtsLibcoreTestCases": [ 278 ("com.android.org.conscrypt.javax.net.ssl.SSLSocketVersionCompatibilityTest#" + m + c) 279 for (m, c) in itertools.product( 280 [ 281 "test_SSLSocket_interrupt_read_withoutAutoClose", 282 "test_SSLSocket_setSoWriteTimeout", 283 ], 284 [ 285 "[0: TLSv1.2 client, TLSv1.2 server]", 286 "[1: TLSv1.2 client, TLSv1.3 server]", 287 "[2: TLSv1.3 client, TLSv1.2 server]", 288 "[3: TLSv1.3 client, TLSv1.3 server]", 289 ] 290 ) 291 ] + [ 292 ("libcore.dalvik.system.DelegateLastClassLoaderTest#" + m) for m in [ 293 "testLookupOrderNodelegate_getResource", 294 "testLookupOrder_getResource", 295 ] 296 ] 297} 298 299# Tests failing because of linking issues, currently exluded from MTS 300# and Mainline Presubmits to minimize noise in continuous runs while 301# we investigate. 302# 303# TODO(b/247108425): Address the linking issues and re-enable these 304# tests. 305failing_tests_excluded_from_mts_and_mainline_presubmits = { 306 "art_standalone_compiler_tests": ["JniCompilerTest*"], 307 "art_standalone_libartpalette_tests": ["PaletteClientJniTest*"], 308} 309 310# Is `run_test` a Checker test (i.e. a test containing Checker 311# assertions)? 312def is_checker_test(run_test): 313 return re.match("^[0-9]+-checker-", run_test) 314 315 316class Generator: 317 def __init__(self, top_dir): 318 """Generator of ART test files for an Android source tree anchored at `top_dir`.""" 319 # Path to the Android top source tree. 320 self.top_dir = top_dir 321 # Path to the ART directory 322 self.art_dir = os.path.join(top_dir, "art") 323 # Path to the ART tests directory. 324 self.art_test_dir = os.path.join(self.art_dir, "test") 325 # Path to the MTS configuration directory. 326 self.mts_config_dir = os.path.join( 327 top_dir, "test", "mts", "tools", "mts-tradefed", "res", "config") 328 329 def enumerate_run_tests(self): 330 return sorted([run_test 331 for run_test in os.listdir(self.art_test_dir) 332 if re.match("^[0-9]{3,}-", run_test)]) 333 334 # Return the metadata of a test, if any. 335 def get_test_metadata(self, run_test): 336 run_test_path = os.path.join(self.art_test_dir, run_test) 337 metadata_file = os.path.join(run_test_path, "test-metadata.json") 338 metadata = {} 339 if os.path.exists(metadata_file): 340 with open(metadata_file, "r") as f: 341 try: 342 metadata = json.load(f) 343 except json.decoder.JSONDecodeError: 344 logging.error(f"Unable to parse test metadata file `{metadata_file}`") 345 raise 346 return metadata 347 348 # Can the build script of `run_test` be safely ignored? 349 def can_ignore_build_script(self, run_test): 350 # Check whether there are test metadata with build parameters 351 # enabling us to safely ignore the build script. 352 metadata = self.get_test_metadata(run_test) 353 build_param = metadata.get("build-param", {}) 354 # Ignore build scripts that are just about preventing building for 355 # the JVM and/or using VarHandles (Soong builds JARs with 356 # VarHandle support by default (i.e. by using an API level greater 357 # or equal to 28), so we can ignore build scripts that just 358 # request support for this feature.) 359 experimental_var_handles = {"experimental": "var-handles"} 360 jvm_supported_false = {"jvm-supported": "false"} 361 if (build_param == experimental_var_handles or 362 build_param == jvm_supported_false or 363 build_param == experimental_var_handles | jvm_supported_false): 364 return True 365 return False 366 367 # Is building `run_test` supported? 368 # TODO(b/147814778): Add build support for more tests. 369 def is_buildable(self, run_test): 370 run_test_path = os.path.join(self.art_test_dir, run_test) 371 372 # Skip tests with non-default build rules, unless these build 373 # rules can be safely ignored. 374 if (os.path.isfile(os.path.join(run_test_path, "generate-sources")) or 375 os.path.isfile(os.path.join(run_test_path, "javac_post.sh"))): 376 return False 377 if os.path.isfile(os.path.join(run_test_path, "build.py")): 378 if not self.can_ignore_build_script(run_test): 379 return False 380 # Skip tests with sources outside the `src` directory. 381 for subdir in ["jasmin", 382 "jasmin-multidex", 383 "smali", 384 "smali-ex", 385 "smali-multidex", 386 "src-aotex", 387 "src-bcpex", 388 "src-ex", 389 "src-ex2", 390 "src-multidex"]: 391 if os.path.isdir(os.path.join(run_test_path, subdir)): 392 return False 393 # Skip tests that have both an `src` directory and an `src-art` directory. 394 if os.path.isdir(os.path.join(run_test_path, "src")) and \ 395 os.path.isdir(os.path.join(run_test_path, "src-art")): 396 return False 397 # Skip tests that have neither an `src` directory nor an `src-art` directory. 398 if not os.path.isdir(os.path.join(run_test_path, "src")) and \ 399 not os.path.isdir(os.path.join(run_test_path, "src-art")): 400 return False 401 # Skip test with a copy of `sun.misc.Unsafe`. 402 if os.path.isfile(os.path.join(run_test_path, "src", "sun", "misc", "Unsafe.java")): 403 return False 404 # Skip tests with Hidden API specs. 405 if os.path.isfile(os.path.join(run_test_path, "hiddenapi-flags.csv")): 406 return False 407 # All other tests are considered buildable. 408 return True 409 410 # Can the run script of `run_test` be safely ignored? 411 def can_ignore_run_script(self, run_test): 412 # Unconditionally consider some identified tests that have a 413 # (not-yet-handled) custom `run` script as runnable. 414 # 415 # TODO(rpl): Get rid of this exception mechanism by supporting 416 # these tests' `run` scripts properly. 417 if run_test in runnable_test_exceptions: 418 return True 419 # Check whether there are test metadata with run parameters 420 # enabling us to safely ignore the run script. 421 metadata = self.get_test_metadata(run_test) 422 run_param = metadata.get("run-param", {}) 423 if run_param.get("default-run", ""): 424 return True 425 return False 426 427 def gen_libs_list_impl(self, library_type, libraries): 428 if len(libraries) == 0: 429 return "" 430 libraries_joined = """, 431 """.join(libraries) 432 return f""" 433 {library_type}: [ 434 {libraries_joined} 435 ],""" 436 437 def gen_libs_list(self, libraries): 438 return self.gen_libs_list_impl("libs", libraries); 439 440 def gen_static_libs_list(self, libraries): 441 return self.gen_libs_list_impl("static_libs", libraries); 442 443 def gen_java_library_rule(self, name, src_dir, libraries): 444 return f"""\ 445 446 447 // Library with {src_dir}/ sources for the test. 448 java_library {{ 449 name: "{name}", 450 defaults: ["art-run-test-defaults"],{self.gen_libs_list(libraries)} 451 srcs: ["{src_dir}/**/*.java"], 452 }}""" 453 454 # Is (successfully) running `run_test` supported? 455 # TODO(b/147812905): Add run-time support for more tests. 456 def is_runnable(self, run_test): 457 run_test_path = os.path.join(self.art_test_dir, run_test) 458 459 # Skip tests with non-default run rules, unless these run rules 460 # can be safely ignored. 461 if os.path.isfile(os.path.join(run_test_path, "run.py")): 462 if not self.can_ignore_run_script(run_test): 463 return False 464 # Skip tests known to fail. 465 if run_test in known_failing_tests: 466 return False 467 # All other tests are considered runnable. 468 return True 469 470 def is_slow(self, run_test): 471 return run_test in known_slow_tests 472 473 def regen_bp_files(self, run_tests, buildable_tests): 474 for run_test in run_tests: 475 # Remove any previously generated file. 476 bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp") 477 if os.path.exists(bp_file): 478 logging.debug(f"Removing `{bp_file}`.") 479 os.remove(bp_file) 480 481 for run_test in buildable_tests: 482 self.regen_bp_file(run_test) 483 484 def regen_bp_file(self, run_test): 485 """Regenerate Blueprint file for an ART run-test.""" 486 487 run_test_path = os.path.join(self.art_test_dir, run_test) 488 bp_file = os.path.join(run_test_path, "Android.bp") 489 490 # Optional test metadata (JSON file). 491 metadata = self.get_test_metadata(run_test) 492 493 run_test_module_name = ART_RUN_TEST_MODULE_NAME_PREFIX + run_test 494 495 # Set the test configuration template. 496 if self.is_runnable(run_test): 497 if "cts" in metadata.get("test_suites", []): 498 test_config_template = "art-run-test-target-cts-template" 499 elif self.is_slow(run_test): 500 test_config_template = "art-run-test-target-slow-template" 501 else: 502 test_config_template = "art-run-test-target-template" 503 else: 504 test_config_template = "art-run-test-target-no-test-suite-tag-template" 505 506 # Define `test_suites`, if present in the test's metadata. 507 test_suites = "" 508 if metadata.get("test_suites"): 509 test_suites = f"""\ 510 511 test_suites: {json.dumps(metadata.get("test_suites"))},""" 512 513 if is_checker_test(run_test): 514 include_src = """\ 515 516 // Include the Java source files in the test's artifacts, to make Checker assertions 517 // available to the TradeFed test runner. 518 include_srcs: true,""" 519 else: 520 include_src = "" 521 522 # The default source directory is `src`, except if `src-art` exists. 523 if os.path.isdir(os.path.join(run_test_path, "src-art")): 524 source_dir = "src-art" 525 else: 526 source_dir = "src" 527 528 src_library_rules = [] 529 test_libraries = [] 530 if os.path.isdir(os.path.join(run_test_path, "src2")): 531 src_library_rules.append(self.gen_java_library_rule( 532 f"{run_test_module_name}-{source_dir}", 533 source_dir, 534 test_libraries)) 535 test_libraries.append(f"\"{run_test_module_name}-src\"") 536 source_dir = "src2" 537 538 with open(bp_file, "w") as f: 539 logging.debug(f"Writing `{bp_file}`.") 540 f.write(textwrap.dedent(f"""\ 541 // {ADVISORY} 542 543 // Build rules for ART run-test `{run_test}`. 544 545 package {{ 546 // See: http://go/android-license-faq 547 // A large-scale-change added 'default_applicable_licenses' to import 548 // all of the 'license_kinds' from "art_license" 549 // to get the below license kinds: 550 // SPDX-license-identifier-Apache-2.0 551 default_applicable_licenses: ["art_license"], 552 }}{''.join(src_library_rules)} 553 554 // Test's Dex code. 555 java_test {{ 556 name: "{run_test_module_name}", 557 defaults: ["art-run-test-defaults"], 558 test_config_template: ":{test_config_template}", 559 srcs: ["{source_dir}/**/*.java"],{self.gen_static_libs_list(test_libraries)} 560 data: [ 561 ":{run_test_module_name}-expected-stdout", 562 ":{run_test_module_name}-expected-stderr", 563 ],{test_suites}{include_src} 564 }} 565 566 // Test's expected standard output. 567 genrule {{ 568 name: "{run_test_module_name}-expected-stdout", 569 out: ["{run_test_module_name}-expected-stdout.txt"], 570 srcs: ["expected-stdout.txt"], 571 cmd: "cp -f $(in) $(out)", 572 }} 573 574 // Test's expected standard error. 575 genrule {{ 576 name: "{run_test_module_name}-expected-stderr", 577 out: ["{run_test_module_name}-expected-stderr.txt"], 578 srcs: ["expected-stderr.txt"], 579 cmd: "cp -f $(in) $(out)", 580 }} 581 """)) 582 583 def regen_test_mapping_file(self, art_run_tests): 584 """Regenerate ART's `TEST_MAPPING`.""" 585 586 run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 587 588 # Mainline presubmits. 589 mainline_presubmit_apex_suffix = "[com.google.android.art.apex]" 590 mainline_other_presubmit_tests = [] 591 mainline_presubmit_tests = (mainline_other_presubmit_tests + run_test_module_names + 592 art_gtests_mainline_presubmit_module_names) 593 mainline_presubmit_tests_dict = [ 594 ({"name": t + mainline_presubmit_apex_suffix, 595 "options": [ 596 {"exclude-filter": e} 597 for e in failing_tests_excluded_from_mts_and_mainline_presubmits[t] 598 ]} 599 if t in failing_tests_excluded_from_mts_and_mainline_presubmits 600 else {"name": t + mainline_presubmit_apex_suffix}) 601 for t in mainline_presubmit_tests 602 ] 603 604 # Android Virtualization Framework presubmits 605 avf_presubmit_tests = ["ComposHostTestCases"] 606 avf_presubmit_tests_dict = [{"name": t} for t in avf_presubmit_tests] 607 608 # Presubmits. 609 other_presubmit_tests = [ 610 "ArtServiceTests", 611 "BootImageProfileTest", 612 "CtsJdwpTestCases", 613 "art-apex-update-rollback", 614 "art_standalone_dexpreopt_tests", 615 ] 616 presubmit_tests = other_presubmit_tests + run_test_module_names + art_gtest_module_names 617 presubmit_tests_dict = [{"name": t} for t in presubmit_tests] 618 hwasan_presubmit_tests_dict = [{"name": t} for t in presubmit_tests 619 if t not in known_failing_on_hwasan_tests] 620 621 # Use an `OrderedDict` container to preserve the order in which items are inserted. 622 # Do not produce an entry for a test group if it is empty. 623 test_mapping_dict = collections.OrderedDict([ 624 (test_group_name, test_group_dict) 625 for (test_group_name, test_group_dict) 626 in [ 627 ("mainline-presubmit", mainline_presubmit_tests_dict), 628 ("presubmit", presubmit_tests_dict), 629 ("hwasan-presubmit", hwasan_presubmit_tests_dict), 630 ("avf-presubmit", avf_presubmit_tests_dict), 631 ] 632 if test_group_dict 633 ]) 634 test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT) 635 636 test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING") 637 with open(test_mapping_file, "w") as f: 638 logging.debug(f"Writing `{test_mapping_file}`.") 639 f.write(f"// {ADVISORY}\n") 640 f.write(test_mapping_contents) 641 f.write("\n") 642 643 def create_mts_test_shard(self, description, tests, shard_num, copyright_year, comments = []): 644 """Factory method instantiating an `MtsTestShard`.""" 645 return self.MtsTestShard(self.mts_config_dir, 646 description, tests, shard_num, copyright_year, comments) 647 648 class MtsTestShard: 649 """Class encapsulating data and generation logic for an ART MTS test shard.""" 650 651 def __init__(self, mts_config_dir, description, tests, shard_num, copyright_year, comments): 652 self.mts_config_dir = mts_config_dir 653 self.description = description 654 self.tests = tests 655 self.shard_num = shard_num 656 self.copyright_year = copyright_year 657 self.comments = comments 658 659 def shard_id(self): 660 return f"{self.shard_num:02}" 661 662 def test_plan_name(self): 663 return "mts-art-shard-" + self.shard_id() 664 665 def test_list_name(self): 666 return "mts-art-tests-list-user-shard-" + self.shard_id() 667 668 def regen_test_plan_file(self): 669 """Regenerate ART MTS test plan file shard (`mts-art-shard-<shard_num>.xml`).""" 670 root = xml.dom.minidom.Document() 671 672 advisory_header = root.createComment(f" {ADVISORY} ") 673 root.appendChild(advisory_header) 674 copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 675 root.appendChild(copyright_header) 676 677 configuration = root.createElement("configuration") 678 root.appendChild(configuration) 679 configuration.setAttribute( 680 "description", 681 f"Run mts-art-shard-{self.shard_id()} from a preexisting MTS installation.") 682 683 # Included XML files. 684 included_xml_files = ["mts", self.test_list_name()] 685 for xml_file in included_xml_files: 686 include = root.createElement("include") 687 include.setAttribute("name", xml_file) 688 configuration.appendChild(include) 689 690 # Test plan name. 691 option = root.createElement("option") 692 option.setAttribute("name", "plan") 693 option.setAttribute("value", self.test_plan_name()) 694 configuration.appendChild(option) 695 696 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 697 698 test_plan_file = os.path.join(self.mts_config_dir, self.test_plan_name() + ".xml") 699 with open(test_plan_file, "wb") as f: 700 logging.debug(f"Writing `{test_plan_file}`.") 701 f.write(xml_str) 702 703 def regen_test_list_file(self): 704 """Regenerate ART MTS test list file (`mts-art-tests-list-user-shard-<shard_num>.xml`).""" 705 root = xml.dom.minidom.Document() 706 707 advisory_header = root.createComment(f" {ADVISORY} ") 708 root.appendChild(advisory_header) 709 copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 710 root.appendChild(copyright_header) 711 712 configuration = root.createElement("configuration") 713 root.appendChild(configuration) 714 configuration.setAttribute( 715 "description", 716 f"List of ART MTS tests that do not need root access (shard {self.shard_id()})" 717 ) 718 719 # Test declarations. 720 # ------------------ 721 722 def append_test_declaration(test): 723 option = root.createElement("option") 724 option.setAttribute("name", "compatibility:include-filter") 725 option.setAttribute("value", test) 726 configuration.appendChild(option) 727 728 test_declarations_comments = [self.description + "."] 729 test_declarations_comments.extend(self.comments) 730 for c in test_declarations_comments: 731 xml_comment = root.createComment(f" {c} ") 732 configuration.appendChild(xml_comment) 733 for t in self.tests: 734 append_test_declaration(t) 735 736 # `MainlineTestModuleController` configurations. 737 # ---------------------------------------------- 738 739 def append_module_controller_configuration(test): 740 option = root.createElement("option") 741 option.setAttribute("name", "compatibility:module-arg") 742 option.setAttribute("value", f"{test}:enable:true") 743 configuration.appendChild(option) 744 745 module_controller_configuration_comments = [ 746 f"Enable MainlineTestModuleController for {self.description}."] 747 module_controller_configuration_comments.extend(self.comments) 748 for c in module_controller_configuration_comments: 749 xml_comment = root.createComment(f" {c} ") 750 configuration.appendChild(xml_comment) 751 for t in self.tests: 752 append_module_controller_configuration(t) 753 754 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 755 756 test_list_file = os.path.join(self.mts_config_dir, self.test_list_name() + ".xml") 757 with open(test_list_file, "wb") as f: 758 logging.debug(f"Writing `{test_list_file}`.") 759 f.write(xml_str) 760 761 def regen_mts_art_tests_list_user_file(self, num_mts_art_run_test_shards): 762 """Regenerate ART MTS test list file (`mts-art-tests-list-user.xml`).""" 763 root = xml.dom.minidom.Document() 764 765 advisory_header = root.createComment(f" {ADVISORY} ") 766 root.appendChild(advisory_header) 767 copyright_header = root.createComment(copyright_header_text(2020)) 768 root.appendChild(copyright_header) 769 770 configuration = root.createElement("configuration") 771 root.appendChild(configuration) 772 configuration.setAttribute("description", "List of ART MTS tests that do not need root access.") 773 774 # Included XML files. 775 for s in range(num_mts_art_run_test_shards): 776 include = root.createElement("include") 777 include.setAttribute("name", f"mts-art-tests-list-user-shard-{s:02}") 778 configuration.appendChild(include) 779 780 def append_test_exclusion(test): 781 option = root.createElement("option") 782 option.setAttribute("name", "compatibility:exclude-filter") 783 option.setAttribute("value", test) 784 configuration.appendChild(option) 785 786 # Excluded flaky tests. 787 xml_comment = root.createComment(" Excluded flaky tests (b/209958457). ") 788 configuration.appendChild(xml_comment) 789 for module in flaky_tests_excluded_from_mts: 790 for testcase in flaky_tests_excluded_from_mts[module]: 791 append_test_exclusion(f"{module} {testcase}") 792 793 # Excluded failing tests. 794 xml_comment = root.createComment(" Excluded failing tests (b/247108425). ") 795 configuration.appendChild(xml_comment) 796 for module in failing_tests_excluded_from_mts_and_mainline_presubmits: 797 for testcase in failing_tests_excluded_from_mts_and_mainline_presubmits[module]: 798 append_test_exclusion(f"{module} {testcase}") 799 800 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 801 802 mts_art_tests_list_user_file = os.path.join(self.mts_config_dir, "mts-art-tests-list-user.xml") 803 with open(mts_art_tests_list_user_file, "wb") as f: 804 logging.debug(f"Writing `{mts_art_tests_list_user_file}`.") 805 f.write(xml_str) 806 807 def regen_art_mts_files(self, art_run_tests): 808 """Regenerate ART MTS definition files.""" 809 810 # Remove any previously MTS ART test plan shard (`mts-art-shard-[0-9]+.xml`) 811 # and any test list shard (`mts-art-tests-list-user-shard-[0-9]+.xml`). 812 old_test_plan_shards = sorted([ 813 test_plan_shard 814 for test_plan_shard in os.listdir(self.mts_config_dir) 815 if re.match("^mts-art-(tests-list-user-)?shard-[0-9]+.xml$", test_plan_shard)]) 816 for shard in old_test_plan_shards: 817 shard_path = os.path.join(self.mts_config_dir, shard) 818 if os.path.exists(shard_path): 819 logging.debug(f"Removing `{shard_path}`.") 820 os.remove(shard_path) 821 822 mts_test_shards = [] 823 824 # ART run-tests shard(s). 825 art_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 826 art_run_test_shards = split_list(art_run_test_module_names, NUM_MTS_ART_RUN_TEST_SHARDS) 827 for i in range(len(art_run_test_shards)): 828 art_tests_shard_i_tests = art_run_test_shards[i] 829 art_tests_shard_i = self.create_mts_test_shard( 830 "ART run-tests", art_tests_shard_i_tests, i, 2020, 831 ["TODO(rpl): Find a way to express this list in a more concise fashion."]) 832 mts_test_shards.append(art_tests_shard_i) 833 834 # CTS Libcore non-OJ tests (`CtsLibcoreTestCases`) shard. 835 cts_libcore_tests_shard_num = len(mts_test_shards) 836 cts_libcore_tests_shard = self.create_mts_test_shard( 837 "CTS Libcore non-OJ tests", ["CtsLibcoreTestCases"], cts_libcore_tests_shard_num, 2020) 838 mts_test_shards.append(cts_libcore_tests_shard) 839 840 # Other CTS Libcore tests shard. 841 other_cts_libcore_tests_shard_num = len(mts_test_shards) 842 other_cts_libcore_tests_shard_tests = [ 843 "CtsLibcoreApiEvolutionTestCases", 844 "CtsLibcoreFileIOTestCases", 845 "CtsLibcoreJsr166TestCases", 846 "CtsLibcoreLegacy22TestCases", 847 "CtsLibcoreOjTestCases", 848 "CtsLibcoreWycheproofBCTestCases", 849 "MtsLibcoreOkHttpTestCases", 850 ] 851 other_cts_libcore_tests_shard = self.create_mts_test_shard( 852 "CTS Libcore OJ tests", other_cts_libcore_tests_shard_tests, 853 other_cts_libcore_tests_shard_num, 2021) 854 mts_test_shards.append(other_cts_libcore_tests_shard) 855 856 # ART gtests shard. 857 # TODO: Also handle the case of gtests requiring root access to the device 858 # (`art_gtest_eng_only_module_names`). 859 art_gtests_shard_num = len(mts_test_shards) 860 art_gtests_shard_tests = art_gtest_mts_user_module_names 861 art_gtests_shard = self.create_mts_test_shard( 862 "ART gtests", art_gtests_shard_tests, art_gtests_shard_num, 2022) 863 mts_test_shards.append(art_gtests_shard) 864 865 for s in mts_test_shards: 866 s.regen_test_plan_file() 867 s.regen_test_list_file() 868 869 self.regen_mts_art_tests_list_user_file(len(mts_test_shards)) 870 871 def regen_test_files(self, regen_art_mts): 872 """Regenerate ART test files. 873 874 Args: 875 regen_art_mts: If true, also regenerate the ART MTS definition. 876 """ 877 run_tests = self.enumerate_run_tests() 878 879 # Create a list of the tests that can currently be built, and for 880 # which a Blueprint file is to be generated. 881 buildable_tests = list(filter(self.is_buildable, run_tests)) 882 883 # Create a list of the tests that can be built and run 884 # (successfully). These tests are to be added to ART's 885 # `TEST_MAPPING` file and also tagged as part of TradeFed's 886 # `art-target-run-test` test suite via the `test-suite-tag` option 887 # in their configuration file. 888 expected_succeeding_tests = list(filter(self.is_runnable, buildable_tests)) 889 890 # Regenerate Blueprint files. 891 # --------------------------- 892 893 self.regen_bp_files(run_tests, buildable_tests) 894 895 buildable_tests_percentage = int(len(buildable_tests) * 100 / len(run_tests)) 896 897 print(f"Generated Blueprint files for {len(buildable_tests)} ART run-tests out of" 898 f" {len(run_tests)} ({buildable_tests_percentage}%).") 899 900 # Regenerate `TEST_MAPPING` file. 901 # ------------------------------- 902 903 # Note: We only include ART run-tests expected to succeed for now. 904 905 num_presubmit_run_tests = len(expected_succeeding_tests) 906 num_mainline_presubmit_run_tests = len(expected_succeeding_tests) 907 908 self.regen_test_mapping_file(expected_succeeding_tests) 909 910 expected_succeeding_tests_percentage = int( 911 len(expected_succeeding_tests) * 100 / len(run_tests)) 912 913 mainline_presubmit_gtests_percentage = int( 914 len(art_gtests_mainline_presubmit_module_names) * 100 / len(art_gtest_module_names)) 915 916 print(f"Generated TEST_MAPPING entries for {len(expected_succeeding_tests)} ART run-tests out" 917 f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%):") 918 for (num_tests, test_kind, tests_percentage, test_group_name) in [ 919 (num_mainline_presubmit_run_tests, "ART run-tests", 100, "mainline-presubmit"), 920 (len(art_gtests_mainline_presubmit_module_names), "ART gtests", 921 mainline_presubmit_gtests_percentage, "mainline-presubmit"), 922 (num_presubmit_run_tests, "ART run-tests", 100, "presubmit"), 923 (len(art_gtest_module_names), "ART gtests", 100, "presubmit"), 924 ]: 925 print( 926 f" {num_tests:3d} {test_kind} ({tests_percentage}%) in `{test_group_name}` test group.") 927 928 # Regenerate ART MTS definition (optional). 929 # ----------------------------------------- 930 931 if regen_art_mts: 932 self.regen_art_mts_files(expected_succeeding_tests) 933 print(f"Generated ART MTS entries for {len(expected_succeeding_tests)} ART run-tests out" 934 f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%).") 935 936def main(): 937 if "ANDROID_BUILD_TOP" not in os.environ: 938 logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?") 939 sys.exit(1) 940 941 parser = argparse.ArgumentParser( 942 formatter_class=argparse.RawDescriptionHelpFormatter, 943 description=textwrap.dedent("Regenerate some ART test related files."), 944 epilog=textwrap.dedent("""\ 945 Regenerate ART run-tests Blueprint files, ART's `TEST_MAPPING` file, and 946 optionally the ART MTS (Mainline Test Suite) definition. 947 """)) 948 parser.add_argument("-m", "--regen-art-mts", help="regenerate the ART MTS definition as well", 949 action="store_true") 950 parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true") 951 args = parser.parse_args() 952 953 if args.verbose: 954 logging.getLogger().setLevel(logging.DEBUG) 955 956 generator = Generator(os.path.join(os.environ["ANDROID_BUILD_TOP"])) 957 generator.regen_test_files(args.regen_art_mts) 958 959 960if __name__ == "__main__": 961 main() 962