1#! /usr/bin/env python3 2# 3# Copyright 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17# Regenerate some ART test related files. 18 19# This script handles only a subset of ART run-tests at the moment; additional 20# cases will be added later. 21 22import argparse 23import copy 24import collections 25import itertools 26import json 27import logging 28import os 29import re 30import sys 31import textwrap 32import xml.dom.minidom 33 34logging.basicConfig(format='%(levelname)s: %(message)s') 35 36ME = os.path.basename(sys.argv[0]) 37 38# Common advisory placed at the top of all generated files. 39ADVISORY = f"Generated by `{ME}`. Do not edit manually." 40 41# Default indentation unit. 42INDENT = " " 43 44# Indentation unit for XML files. 45XML_INDENT = " " 46 47def reindent(str, indent = ""): 48 """Reindent literal string while removing common leading spaces.""" 49 return textwrap.indent(textwrap.dedent(str), indent) 50 51def copyright_header_text(year): 52 """Return the copyright header text used in XML files.""" 53 return reindent(f"""\ 54 Copyright (C) {year} The Android Open Source Project 55 56 Licensed under the Apache License, Version 2.0 (the "License"); 57 you may not use this file except in compliance with the License. 58 You may obtain a copy of the License at 59 60 http://www.apache.org/licenses/LICENSE-2.0 61 62 Unless required by applicable law or agreed to in writing, software 63 distributed under the License is distributed on an "AS IS" BASIS, 64 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 65 See the License for the specific language governing permissions and 66 limitations under the License. 67 """, " ") 68 69def split_list(l, n): 70 """Return a list of `n` sublists of (contiguous) elements of list `l`.""" 71 assert n > 0 72 (d, m) = divmod(len(l), n) 73 # If the length of `l` is divisible by `n`, use that that divisor (`d`) as size of each sublist; 74 # otherwise, the next integer value (`d + 1`). 75 s = d if m == 0 else d + 1 76 result = [l[i:i + s] for i in range(0, len(l), s)] 77 assert len(result) == n 78 return result 79 80# The prefix used in the Soong module name of all ART run-tests. 81ART_RUN_TEST_MODULE_NAME_PREFIX = "art-run-test-" 82 83# Number of shards used to declare ART run-tests in the sharded ART MTS test plan. 84NUM_MTS_ART_RUN_TEST_SHARDS = 1 85 86# Curated list of tests that have a custom `run` script, but that are 87# known to work fine with the default test execution strategy (i.e. 88# when ignoring their `run` script), even if not exactly as they would 89# with the original ART run-test harness. 90runnable_test_exceptions = frozenset([ 91 "055-enum-performance", 92 "059-finalizer-throw", 93 "080-oom-throw", 94 "1004-checker-volatile-ref-load", 95 "133-static-invoke-super", 96 "1338-gc-no-los", 97 "151-OpenFileLimit", 98 "159-app-image-fields", 99 "160-read-barrier-stress", 100 "163-app-image-methods", 101 "165-lock-owner-proxy", 102 "168-vmstack-annotated", 103 "176-app-image-string", 104 "2232-write-metrics-to-log", 105 "304-method-tracing", 106 "628-vdex", 107 "643-checker-bogus-ic", 108 "676-proxy-jit-at-first-use", 109 "677-fsi2", 110 "678-quickening", 111 "818-clinit-nterp", 112 "821-madvise-willneed", 113]) 114 115known_slow_tests = frozenset([ 116 "175-alloc-big-bignums", 117]) 118 119# Known failing ART run-tests. 120# TODO(rpl): Investigate and address the causes of failures. 121known_failing_tests = frozenset([ 122 "004-SignalTest", 123 "004-UnsafeTest", 124 "051-thread", 125 "086-null-super", 126 "087-gc-after-link", 127 # 1002-notify-startup: Dependency on `libarttest` + custom `check` script. 128 "1002-notify-startup", 129 "1337-gc-coverage", 130 "1339-dead-reference-safe", 131 "136-daemon-jni-shutdown", 132 "139-register-natives", 133 "148-multithread-gc-annotations", 134 "149-suspend-all-stress", 135 "150-loadlibrary", 136 "154-gc-loop", 137 "169-threadgroup-jni", 138 "177-visibly-initialized-deadlock", 139 "179-nonvirtual-jni", 140 "1945-proxy-method-arguments", 141 "2011-stack-walk-concurrent-instrument", 142 # 2040-huge-native-alloc: Fails with: 143 # 144 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 145 # at Main.main(Main.java:56) 146 # 147 "2040-huge-native-alloc", 148 "203-multi-checkpoint", 149 "2033-shutdown-mechanics", 150 "2036-jni-filechannel", 151 "2037-thread-name-inherit", 152 "2235-JdkUnsafeTest", 153 "305-other-fault-handler", 154 # 449-checker-bce: Dependency on `libarttest`. 155 "449-checker-bce", 156 "454-get-vreg", 157 "461-get-reference-vreg", 158 "466-get-live-vreg", 159 "497-inlining-and-class-loader", 160 "530-regression-lse", 161 "555-UnsafeGetLong-regression", 162 # 596-monitor-inflation: Dependency on `libarttest`. 163 "596-monitor-inflation", 164 "602-deoptimizeable", 165 "604-hot-static-interface", 166 "616-cha-native", 167 "616-cha-regression-proxy-method", 168 # 623-checker-loop-regressions: Dependency on `libarttest`. 169 "623-checker-loop-regressions", 170 "626-set-resolved-string", 171 "642-fp-callees", 172 "647-jni-get-field-id", 173 "655-jit-clinit", 174 "656-loop-deopt", 175 "664-aget-verifier", 176 # 680-checker-deopt-dex-pc-0: Dependency on `libarttest`. 177 "680-checker-deopt-dex-pc-0", 178 "685-deoptimizeable", 179 "687-deopt", 180 "693-vdex-inmem-loader-evict", 181 "708-jit-cache-churn", 182 # 716-jli-jit-samples: Dependency on `libarttest`. 183 "716-jli-jit-samples", 184 "717-integer-value-of", 185 "720-thread-priority", 186 # 730-cha-deopt: Fails with: 187 # 188 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 189 # at Main.main(Main.java:24) 190 # 191 "730-cha-deopt", 192 # 813-fp-args: Dependency on `libarttest`. 193 "813-fp-args", 194 # 821-many-args: Fails with: 195 # 196 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 197 # at Main.main(Main.java:20) 198 # 199 "821-many-args", 200 # 826-infinite-loop: The test expects an argument passed to `Main.main` (the test library, 201 # usually `arttestd` or `arttest)`, but the ART run-test TradeFed test runner 202 # (`com.android.tradefed.testtype.ArtRunTest`) does not implement this yet. 203 "826-infinite-loop", 204 # 832-cha-recursive: Dependency on `libarttest`. 205 "832-cha-recursive", 206]) 207 208# ART gtests that do not need root access to the device. 209art_gtest_user_module_names = [ 210 "art_libnativebridge_cts_tests", 211 "art_standalone_cmdline_tests", 212 ###"art_standalone_compiler_tests", # b/275289981 213 # Temporarily disable this test as it is failing with ART module prebuilts (see b/243510263). 214 ### "art_standalone_dex2oat_tests", 215 ###"art_standalone_dexdump_tests", # b/275289981 216 ###"art_standalone_dexlist_tests", # b/275289981 217 # Temporarily disable this test as it is failing with ART module prebuilts (see b/243507635). 218 ### "art_standalone_libartbase_tests", 219 "art_standalone_libartpalette_tests", 220 "art_standalone_libartservice_tests", 221 "art_standalone_libarttools_tests", 222 "art_standalone_libdexfile_support_tests", 223 "art_standalone_libdexfile_tests", 224 "art_standalone_libprofile_tests", 225 ###"art_standalone_oatdump_tests", # b/275289981 226 ###"art_standalone_odrefresh_tests", # b/275289981 227 ###"art_standalone_runtime_compiler_tests", # b/275289981 228 ###"art_standalone_runtime_tests", # b/275289981 229 "art_standalone_sigchain_tests", 230 "libnativeloader_test", 231] 232 233# ART gtests that need root access to the device. 234art_gtest_eng_only_module_names = [ 235 ###"art_standalone_dexoptanalyzer_tests", # b/275289981 236 ###"art_standalone_profman_tests", # b/275289981 237] 238 239# All supported ART gtests. 240art_gtest_module_names = sorted(art_gtest_user_module_names + art_gtest_eng_only_module_names) 241 242# ART gtests supported in MTS that do not need root access to the device. 243art_gtest_mts_user_module_names = copy.copy(art_gtest_user_module_names) 244# Temporarily disable `art_standalone_odrefresh_tests` in MTS, 245# as it is currently failing in Mainline testing 246# (b/206335809); a fix is in the works but may take some time 247# to land. 248# 249# TODO(b/206335809): Re-enable this test when the fix has landed. 250if "art_standalone_odrefresh_tests" in art_gtest_mts_user_module_names: 251 art_gtest_mts_user_module_names.remove("art_standalone_odrefresh_tests") 252 253# ART gtests supported in Mainline presubmits. 254art_gtests_mainline_presubmit_module_names = copy.copy(art_gtest_module_names) 255# Temporarily disable `art_standalone_odrefresh_tests` in Mainline 256# presubmits, as it is currently failing in Mainline testing 257# (b/206335809); a fix is in the works but may take some time to 258# land. 259# 260# TODO(b/206335809): Re-enable this test when the fix has landed. 261if "art_standalone_odrefresh_tests" in art_gtests_mainline_presubmit_module_names: 262 art_gtests_mainline_presubmit_module_names.remove("art_standalone_odrefresh_tests") 263 264# Tests exhibiting a flaky behavior, currently exluded from MTS for 265# the stake of stability / confidence (b/209958457). 266flaky_tests_excluded_from_mts = [ 267 ("CtsLibcoreFileIOTestCases" + 268 " android.cts.FileChannelInterProcessLockTest#" + m) for m in [ 269 "test_lockJJZ_Exclusive_asyncChannel", 270 "test_lockJJZ_Exclusive_syncChannel", 271 "test_lock_differentChannelTypes", 272 "test_lockJJZ_Shared_asyncChannel", 273 "test_lockJJZ_Shared_syncChannel", 274 ] 275] + [ 276 ("CtsLibcoreTestCases" + 277 " com.android.org.conscrypt.javax.net.ssl.SSLSocketVersionCompatibilityTest#" + m + c) 278 for (m, c) in itertools.product( 279 [ 280 "test_SSLSocket_interrupt_read_withoutAutoClose", 281 "test_SSLSocket_setSoWriteTimeout", 282 ], 283 [ 284 "[0: TLSv1.2 client, TLSv1.2 server]", 285 "[1: TLSv1.2 client, TLSv1.3 server]", 286 "[2: TLSv1.3 client, TLSv1.2 server]", 287 "[3: TLSv1.3 client, TLSv1.3 server]", 288 ] 289 ) 290] + [ 291 ("CtsLibcoreTestCases" + 292 " libcore.dalvik.system.DelegateLastClassLoaderTest#" + m) for m in [ 293 "testLookupOrderNodelegate_getResource", 294 "testLookupOrder_getResource", 295 ] 296] 297 298# Is `run_test` a Checker test (i.e. a test containing Checker 299# assertions)? 300def is_checker_test(run_test): 301 return re.match("^[0-9]+-checker-", run_test) 302 303 304class Generator: 305 def __init__(self, top_dir): 306 """Generator of ART test files for an Android source tree anchored at `top_dir`.""" 307 # Path to the Android top source tree. 308 self.top_dir = top_dir 309 # Path to the ART directory 310 self.art_dir = os.path.join(top_dir, "art") 311 # Path to the ART tests directory. 312 self.art_test_dir = os.path.join(self.art_dir, "test") 313 # Path to the MTS configuration directory. 314 self.mts_config_dir = os.path.join( 315 top_dir, "test", "mts", "tools", "mts-tradefed", "res", "config") 316 317 def enumerate_run_tests(self): 318 return sorted([run_test 319 for run_test in os.listdir(self.art_test_dir) 320 if re.match("^[0-9]{3,}-", run_test)]) 321 322 # Read build file (Bash script) and return a canonized version of it 323 # (without comments, blank lines, "debugging" statements, etc.). 324 def canonize_build_script(self, build_file): 325 326 def is_comment(line): 327 return re.match("^\\s*#", line) 328 329 def is_blank(line): 330 return re.match("^\\s*$", line) 331 332 # Is `line` a `set -e` statement? 333 def is_set_e(line): 334 return re.match("^\\s*set -e\\s*", line) 335 336 # Should `line` be kept in the canonized build script? 337 def keep_line(line): 338 return not (is_comment(line) or is_blank(line) or is_set_e(line)) 339 340 with open(build_file, "r") as f: 341 lines = f.readlines() 342 return list(filter(keep_line, lines)) 343 344 # Can the build script in `build_file` be safely ignored? 345 def can_ignore_build_script(self, build_file): 346 build_script = self.canonize_build_script(build_file) 347 if len(build_script) == 1: 348 if build_script[0] == "./default-build \"$@\" --experimental var-handles\n": 349 # Soong builds JARs with VarHandle support by default (i.e. by 350 # using an API level greater or equal to 28), so we can ignore 351 # build scripts that just request support for this feature. 352 return True 353 return False 354 355 # Is building `run_test` supported? 356 # TODO(b/147814778): Add build support for more tests. 357 def is_buildable(self, run_test): 358 run_test_path = os.path.join(self.art_test_dir, run_test) 359 360 # Skip tests with non-default build rules, unless these build 361 # rules can be safely ignored. 362 if os.path.isfile(os.path.join(run_test_path, "build")): 363 if not self.can_ignore_build_script(os.path.join(run_test_path, "build")): 364 return False 365 # Skip tests with sources outside the `src` directory. 366 for subdir in ["jasmin", 367 "jasmin-multidex", 368 "smali", 369 "smali-ex", 370 "smali-multidex", 371 "src-aotex", 372 "src-bcpex", 373 "src-ex", 374 "src-ex2", 375 "src-multidex", 376 "src2"]: 377 if os.path.isdir(os.path.join(run_test_path, subdir)): 378 return False 379 # Skip tests that have both an `src` directory and an `src-art` directory. 380 if os.path.isdir(os.path.join(run_test_path, "src")) and \ 381 os.path.isdir(os.path.join(run_test_path, "src-art")): 382 return False 383 # Skip tests that have neither an `src` directory nor an `src-art` directory. 384 if not os.path.isdir(os.path.join(run_test_path, "src")) and \ 385 not os.path.isdir(os.path.join(run_test_path, "src-art")): 386 return False 387 # Skip test with a copy of `sun.misc.Unsafe`. 388 if os.path.isfile(os.path.join(run_test_path, "src", "sun", "misc", "Unsafe.java")): 389 return False 390 # Skip tests with Hidden API specs. 391 if os.path.isfile(os.path.join(run_test_path, "hiddenapi-flags.csv")): 392 return False 393 # All other tests are considered buildable. 394 return True 395 396 # Is (successfully) running `run_test` supported? 397 # TODO(b/147812905): Add run-time support for more tests. 398 def is_runnable(self, run_test): 399 run_test_path = os.path.join(self.art_test_dir, run_test) 400 # Unconditionally consider some identified tests that have a 401 # (not-yet-handled) custom `run` script as runnable. 402 # TODO(rpl): Get rid of this exception mechanism by supporting 403 # these tests' `run` scripts properly. 404 if run_test in runnable_test_exceptions: 405 return True 406 # Skip tests with a custom `run` script. 407 if os.path.isfile(os.path.join(run_test_path, "run")): 408 return False 409 # Skip tests known to fail. 410 if run_test in known_failing_tests: 411 return False 412 # All other tests are considered runnable. 413 return True 414 415 def is_slow(self, run_test): 416 return run_test in known_slow_tests 417 418 def regen_bp_files(self, run_tests, buildable_tests): 419 for run_test in run_tests: 420 # Remove any previously generated file. 421 bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp") 422 if os.path.exists(bp_file): 423 logging.debug(f"Removing `{bp_file}`.") 424 os.remove(bp_file) 425 426 for run_test in buildable_tests: 427 self.regen_bp_file(run_test) 428 429 def regen_bp_file(self, run_test): 430 """Regenerate Blueprint file for an ART run-test.""" 431 432 run_test_path = os.path.join(self.art_test_dir, run_test) 433 bp_file = os.path.join(run_test_path, "Android.bp") 434 435 # Optional test metadata (JSON file). 436 metadata_file = os.path.join(run_test_path, "test-metadata.json") 437 metadata = {} 438 if os.path.exists(metadata_file): 439 with open(metadata_file, "r") as f: 440 metadata = json.load(f) 441 442 run_test_module_name = ART_RUN_TEST_MODULE_NAME_PREFIX + run_test 443 444 # Set the test configuration template. 445 if self.is_runnable(run_test): 446 if "cts" in metadata.get("test_suites", []): 447 test_config_template = "art-run-test-target-cts-template" 448 elif self.is_slow(run_test): 449 test_config_template = "art-run-test-target-slow-template" 450 else: 451 test_config_template = "art-run-test-target-template" 452 else: 453 test_config_template = "art-run-test-target-no-test-suite-tag-template" 454 455 # Define `test_suites`, if present in the test's metadata. 456 test_suites = "" 457 if metadata.get("test_suites"): 458 test_suites = f"""\ 459 460 test_suites: {json.dumps(metadata.get("test_suites"))},""" 461 462 if is_checker_test(run_test): 463 include_src = """\ 464 465 // Include the Java source files in the test's artifacts, to make Checker assertions 466 // available to the TradeFed test runner. 467 include_srcs: true,""" 468 else: 469 include_src = "" 470 471 # The default source directory is `src`, except if `src-art` exists. 472 if os.path.isdir(os.path.join(run_test_path, "src-art")): 473 source_dir = "src-art" 474 else: 475 source_dir = "src" 476 477 with open(bp_file, "w") as f: 478 logging.debug(f"Writing `{bp_file}`.") 479 f.write(textwrap.dedent(f"""\ 480 // {ADVISORY} 481 482 // Build rules for ART run-test `{run_test}`. 483 484 package {{ 485 // See: http://go/android-license-faq 486 // A large-scale-change added 'default_applicable_licenses' to import 487 // all of the 'license_kinds' from "art_license" 488 // to get the below license kinds: 489 // SPDX-license-identifier-Apache-2.0 490 default_applicable_licenses: ["art_license"], 491 }} 492 493 // Test's Dex code. 494 java_test {{ 495 name: "{run_test_module_name}", 496 defaults: ["art-run-test-defaults"], 497 test_config_template: ":{test_config_template}", 498 srcs: ["{source_dir}/**/*.java"], 499 data: [ 500 ":{run_test_module_name}-expected-stdout", 501 ":{run_test_module_name}-expected-stderr", 502 ],{test_suites}{include_src} 503 }} 504 505 // Test's expected standard output. 506 genrule {{ 507 name: "{run_test_module_name}-expected-stdout", 508 out: ["{run_test_module_name}-expected-stdout.txt"], 509 srcs: ["expected-stdout.txt"], 510 cmd: "cp -f $(in) $(out)", 511 }} 512 513 // Test's expected standard error. 514 genrule {{ 515 name: "{run_test_module_name}-expected-stderr", 516 out: ["{run_test_module_name}-expected-stderr.txt"], 517 srcs: ["expected-stderr.txt"], 518 cmd: "cp -f $(in) $(out)", 519 }} 520 """)) 521 522 def regen_test_mapping_file(self, art_run_tests): 523 """Regenerate ART's `TEST_MAPPING`.""" 524 525 run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 526 527 # Mainline presubmits. 528 mainline_other_presubmit_tests = [ 529 "ComposHostTestCases", 530 ] 531 mainline_presubmit_tests = (mainline_other_presubmit_tests + run_test_module_names + 532 art_gtests_mainline_presubmit_module_names) 533 mainline_presubmit_tests_with_apex = [t + "[com.google.android.art.apex]" 534 for t 535 in mainline_presubmit_tests] 536 mainline_presubmit_tests_dict = [{"name": t} for t in mainline_presubmit_tests_with_apex] 537 538 # Presubmits. 539 other_presubmit_tests = [ 540 "CtsJdwpTestCases", 541 "BootImageProfileTest", 542 "ArtServiceTests", 543 "ComposHostTestCases", 544 "art_standalone_dexpreopt_tests", 545 ] 546 presubmit_tests = other_presubmit_tests + run_test_module_names + art_gtest_module_names 547 presubmit_tests_dict = [{"name": t} for t in presubmit_tests] 548 549 # Use an `OrderedDict` container to preserve the order in which items are inserted. 550 # Do not produce an entry for a test group if it is empty. 551 test_mapping_dict = collections.OrderedDict([ 552 (test_group_name, test_group_dict) 553 for (test_group_name, test_group_dict) 554 in [ 555 ("mainline-presubmit", mainline_presubmit_tests_dict), 556 ("presubmit", presubmit_tests_dict), 557 ] 558 if test_group_dict 559 ]) 560 test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT) 561 562 test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING") 563 with open(test_mapping_file, "w") as f: 564 logging.debug(f"Writing `{test_mapping_file}`.") 565 f.write(f"// {ADVISORY}\n") 566 f.write(test_mapping_contents) 567 f.write("\n") 568 569 def create_mts_test_shard(self, description, tests, shard_num, copyright_year, comments = []): 570 """Factory method instantiating an `MtsTestShard`.""" 571 return self.MtsTestShard(self.mts_config_dir, 572 description, tests, shard_num, copyright_year, comments) 573 574 class MtsTestShard: 575 """Class encapsulating data and generation logic for an ART MTS test shard.""" 576 577 def __init__(self, mts_config_dir, description, tests, shard_num, copyright_year, comments): 578 self.mts_config_dir = mts_config_dir 579 self.description = description 580 self.tests = tests 581 self.shard_num = shard_num 582 self.copyright_year = copyright_year 583 self.comments = comments 584 585 def shard_id(self): 586 return f"{self.shard_num:02}" 587 588 def test_plan_name(self): 589 return "mts-art-shard-" + self.shard_id() 590 591 def test_list_name(self): 592 return "mts-art-tests-list-user-shard-" + self.shard_id() 593 594 def regen_test_plan_file(self): 595 """Regenerate ART MTS test plan file shard (`mts-art-shard-<shard_num>.xml`).""" 596 root = xml.dom.minidom.Document() 597 598 advisory_header = root.createComment(f" {ADVISORY} ") 599 root.appendChild(advisory_header) 600 copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 601 root.appendChild(copyright_header) 602 603 configuration = root.createElement("configuration") 604 root.appendChild(configuration) 605 configuration.setAttribute( 606 "description", 607 f"Run mts-art-shard-{self.shard_id()} from a preexisting MTS installation.") 608 609 # Included XML files. 610 included_xml_files = ["mts", self.test_list_name()] 611 for xml_file in included_xml_files: 612 include = root.createElement("include") 613 include.setAttribute("name", xml_file) 614 configuration.appendChild(include) 615 616 # Test plan name. 617 option = root.createElement("option") 618 option.setAttribute("name", "plan") 619 option.setAttribute("value", self.test_plan_name()) 620 configuration.appendChild(option) 621 622 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 623 624 test_plan_file = os.path.join(self.mts_config_dir, self.test_plan_name() + ".xml") 625 with open(test_plan_file, "wb") as f: 626 logging.debug(f"Writing `{test_plan_file}`.") 627 f.write(xml_str) 628 629 def regen_test_list_file(self): 630 """Regenerate ART MTS test list file (`mts-art-tests-list-user-shard-<shard_num>.xml`).""" 631 root = xml.dom.minidom.Document() 632 633 advisory_header = root.createComment(f" {ADVISORY} ") 634 root.appendChild(advisory_header) 635 copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 636 root.appendChild(copyright_header) 637 638 configuration = root.createElement("configuration") 639 root.appendChild(configuration) 640 configuration.setAttribute( 641 "description", 642 f"List of ART MTS tests that do not need root access (shard {self.shard_id()})" 643 ) 644 645 # Test declarations. 646 # ------------------ 647 648 def append_test_declaration(test): 649 option = root.createElement("option") 650 option.setAttribute("name", "compatibility:include-filter") 651 option.setAttribute("value", test) 652 configuration.appendChild(option) 653 654 test_declarations_comments = [self.description + "."] 655 test_declarations_comments.extend(self.comments) 656 for c in test_declarations_comments: 657 xml_comment = root.createComment(f" {c} ") 658 configuration.appendChild(xml_comment) 659 for t in self.tests: 660 append_test_declaration(t) 661 662 # `MainlineTestModuleController` configurations. 663 # ---------------------------------------------- 664 665 def append_module_controller_configuration(test): 666 option = root.createElement("option") 667 option.setAttribute("name", "compatibility:module-arg") 668 option.setAttribute("value", f"{test}:enable:true") 669 configuration.appendChild(option) 670 671 module_controller_configuration_comments = [ 672 f"Enable MainlineTestModuleController for {self.description}."] 673 module_controller_configuration_comments.extend(self.comments) 674 for c in module_controller_configuration_comments: 675 xml_comment = root.createComment(f" {c} ") 676 configuration.appendChild(xml_comment) 677 for t in self.tests: 678 append_module_controller_configuration(t) 679 680 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 681 682 test_list_file = os.path.join(self.mts_config_dir, self.test_list_name() + ".xml") 683 with open(test_list_file, "wb") as f: 684 logging.debug(f"Writing `{test_list_file}`.") 685 f.write(xml_str) 686 687 def regen_mts_art_tests_list_user_file(self, num_mts_art_run_test_shards): 688 """Regenerate ART MTS test list file (`mts-art-tests-list-user.xml`).""" 689 root = xml.dom.minidom.Document() 690 691 advisory_header = root.createComment(f" {ADVISORY} ") 692 root.appendChild(advisory_header) 693 copyright_header = root.createComment(copyright_header_text(2020)) 694 root.appendChild(copyright_header) 695 696 configuration = root.createElement("configuration") 697 root.appendChild(configuration) 698 configuration.setAttribute("description", "List of ART MTS tests that do not need root access.") 699 700 # Included XML files. 701 for s in range(num_mts_art_run_test_shards): 702 include = root.createElement("include") 703 include.setAttribute("name", f"mts-art-tests-list-user-shard-{s:02}") 704 configuration.appendChild(include) 705 706 # Excluded flaky tests. 707 xml_comment = root.createComment(f" Excluded flaky tests (b/209958457). ") 708 configuration.appendChild(xml_comment) 709 710 def append_test_exclusion(test): 711 option = root.createElement("option") 712 option.setAttribute("name", "compatibility:exclude-filter") 713 option.setAttribute("value", test) 714 configuration.appendChild(option) 715 716 for t in flaky_tests_excluded_from_mts: 717 append_test_exclusion(t) 718 719 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 720 721 mts_art_tests_list_user_file = os.path.join(self.mts_config_dir, "mts-art-tests-list-user.xml") 722 with open(mts_art_tests_list_user_file, "wb") as f: 723 logging.debug(f"Writing `{mts_art_tests_list_user_file}`.") 724 f.write(xml_str) 725 726 def regen_art_mts_files(self, art_run_tests): 727 """Regenerate ART MTS definition files.""" 728 729 # Remove any previously MTS ART test plan shard (`mts-art-shard-[0-9]+.xml`) 730 # and any test list shard (`mts-art-tests-list-user-shard-[0-9]+.xml`). 731 old_test_plan_shards = sorted([ 732 test_plan_shard 733 for test_plan_shard in os.listdir(self.mts_config_dir) 734 if re.match("^mts-art-(tests-list-user-)?shard-[0-9]+.xml$", test_plan_shard)]) 735 for shard in old_test_plan_shards: 736 shard_path = os.path.join(self.mts_config_dir, shard) 737 if os.path.exists(shard_path): 738 logging.debug(f"Removing `{shard_path}`.") 739 os.remove(shard_path) 740 741 mts_test_shards = [] 742 743 # ART test (gtest & run-test) shard(s). 744 # TODO: Also handle the case of gtests requiring root access to the device 745 # (`art_gtest_eng_only_module_names`). 746 art_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 747 art_run_test_shards = split_list(art_run_test_module_names, NUM_MTS_ART_RUN_TEST_SHARDS) 748 for i in range(len(art_run_test_shards)): 749 art_tests_shard_i_tests = art_run_test_shards[i] 750 # Append ART gtests to the last ART run-test shard for now. 751 # If needed, consider moving them to their own shard to increase 752 # the parallelization of code coverage runs. 753 if i + 1 == len(art_run_test_shards): 754 art_tests_shard_i_tests.extend(art_gtest_mts_user_module_names) 755 art_tests_shard_i = self.create_mts_test_shard( 756 "ART run-tests", art_tests_shard_i_tests, i, 2020, 757 ["TODO(rpl): Find a way to express this list in a more concise fashion."]) 758 mts_test_shards.append(art_tests_shard_i) 759 760 # CTS Libcore non-OJ tests (`CtsLibcoreTestCases`) shard. 761 cts_libcore_tests_shard_num = len(mts_test_shards) 762 cts_libcore_tests_shard = self.create_mts_test_shard( 763 "CTS Libcore non-OJ tests", ["CtsLibcoreTestCases"], cts_libcore_tests_shard_num, 2020) 764 mts_test_shards.append(cts_libcore_tests_shard) 765 766 # Other CTS Libcore tests shard. 767 other_cts_libcore_tests_shard_num = len(mts_test_shards) 768 other_cts_libcore_tests_shard_tests = [ 769 "CtsLibcoreApiEvolutionTestCases", 770 "CtsLibcoreFileIOTestCases", 771 "CtsLibcoreJsr166TestCases", 772 "CtsLibcoreLegacy22TestCases", 773 "CtsLibcoreOjTestCases", 774 "CtsLibcoreWycheproofBCTestCases", 775 "MtsLibcoreOkHttpTestCases", 776 ] 777 other_cts_libcore_tests_shard = self.create_mts_test_shard( 778 "CTS Libcore OJ tests", other_cts_libcore_tests_shard_tests, 779 other_cts_libcore_tests_shard_num, 2021) 780 mts_test_shards.append(other_cts_libcore_tests_shard) 781 782 for s in mts_test_shards: 783 s.regen_test_plan_file() 784 s.regen_test_list_file() 785 786 self.regen_mts_art_tests_list_user_file(len(mts_test_shards)) 787 788 def regen_test_files(self, regen_art_mts): 789 """Regenerate ART test files. 790 791 Args: 792 regen_art_mts: If true, also regenerate the ART MTS definition. 793 """ 794 run_tests = self.enumerate_run_tests() 795 796 # Create a list of the tests that can currently be built, and for 797 # which a Blueprint file is to be generated. 798 buildable_tests = list(filter(self.is_buildable, run_tests)) 799 800 # Create a list of the tests that can be built and run 801 # (successfully). These tests are to be added to ART's 802 # `TEST_MAPPING` file and also tagged as part of TradeFed's 803 # `art-target-run-test` test suite via the `test-suite-tag` option 804 # in their configuration file. 805 expected_succeeding_tests = list(filter(self.is_runnable, buildable_tests)) 806 807 # Regenerate Blueprint files. 808 # --------------------------- 809 810 self.regen_bp_files(run_tests, buildable_tests) 811 812 buildable_tests_percentage = int(len(buildable_tests) * 100 / len(run_tests)) 813 814 print(f"Generated Blueprint files for {len(buildable_tests)} ART run-tests out of" 815 f" {len(run_tests)} ({buildable_tests_percentage}%).") 816 817 # Regenerate `TEST_MAPPING` file. 818 # ------------------------------- 819 820 # Note: We only include ART run-tests expected to succeed for now. 821 822 num_presubmit_run_tests = len(expected_succeeding_tests) 823 num_mainline_presubmit_run_tests = len(expected_succeeding_tests) 824 825 self.regen_test_mapping_file(expected_succeeding_tests) 826 827 expected_succeeding_tests_percentage = int( 828 len(expected_succeeding_tests) * 100 / len(run_tests)) 829 830 mainline_presubmit_gtests_percentage = int( 831 len(art_gtests_mainline_presubmit_module_names) * 100 / len(art_gtest_module_names)) 832 833 print(f"Generated TEST_MAPPING entries for {len(expected_succeeding_tests)} ART run-tests out" 834 f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%):") 835 for (num_tests, test_kind, tests_percentage, test_group_name) in [ 836 (num_mainline_presubmit_run_tests, "ART run-tests", 100, "mainline-presubmit"), 837 (len(art_gtests_mainline_presubmit_module_names), "ART gtests", 838 mainline_presubmit_gtests_percentage, "mainline-presubmit"), 839 (num_presubmit_run_tests, "ART run-tests", 100, "presubmit"), 840 (len(art_gtest_module_names), "ART gtests", 100, "presubmit"), 841 ]: 842 print( 843 f" {num_tests:3d} {test_kind} ({tests_percentage}%) in `{test_group_name}` test group.") 844 845 # Regenerate ART MTS definition (optional). 846 # ----------------------------------------- 847 848 if regen_art_mts: 849 self.regen_art_mts_files(expected_succeeding_tests) 850 print(f"Generated ART MTS entries for {len(expected_succeeding_tests)} ART run-tests out" 851 f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%).") 852 853def main(): 854 if "ANDROID_BUILD_TOP" not in os.environ: 855 logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?") 856 sys.exit(1) 857 858 parser = argparse.ArgumentParser( 859 formatter_class=argparse.RawDescriptionHelpFormatter, 860 description=textwrap.dedent("Regenerate some ART test related files."), 861 epilog=textwrap.dedent("""\ 862 Regenerate ART run-tests Blueprint files, ART's `TEST_MAPPING` file, and 863 optionally the ART MTS (Mainline Test Suite) definition. 864 """)) 865 parser.add_argument("-m", "--regen-art-mts", help="regenerate the ART MTS definition as well", 866 action="store_true") 867 parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true") 868 args = parser.parse_args() 869 870 if args.verbose: 871 logging.getLogger().setLevel(logging.DEBUG) 872 873 generator = Generator(os.path.join(os.environ["ANDROID_BUILD_TOP"])) 874 generator.regen_test_files(args.regen_art_mts) 875 876 877if __name__ == "__main__": 878 main() 879