• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/env python3
2#
3# Copyright 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# Regenerate some ART test related files.
18
19# This script handles only a subset of ART run-tests at the moment; additional
20# cases will be added later.
21
22import argparse
23import copy
24import collections
25import itertools
26import json
27import logging
28import os
29import re
30import sys
31import textwrap
32import xml.dom.minidom
33
34logging.basicConfig(format='%(levelname)s: %(message)s')
35
36ME = os.path.basename(sys.argv[0])
37
38# Common advisory placed at the top of all generated files.
39ADVISORY = f"Generated by `{ME}`. Do not edit manually."
40
41# Default indentation unit.
42INDENT = "  "
43
44# Indentation unit for XML files.
45XML_INDENT = "    "
46
47def reindent(str, indent = ""):
48  """Reindent literal string while removing common leading spaces."""
49  return textwrap.indent(textwrap.dedent(str), indent)
50
51def copyright_header_text(year):
52  """Return the copyright header text used in XML files."""
53  return reindent(f"""\
54    Copyright (C) {year} The Android Open Source Project
55
56        Licensed under the Apache License, Version 2.0 (the "License");
57        you may not use this file except in compliance with the License.
58        You may obtain a copy of the License at
59
60             http://www.apache.org/licenses/LICENSE-2.0
61
62        Unless required by applicable law or agreed to in writing, software
63        distributed under the License is distributed on an "AS IS" BASIS,
64        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
65        See the License for the specific language governing permissions and
66        limitations under the License.
67    """, " ")
68
69def split_list(l, n):
70  """Return a list of `n` sublists of (contiguous) elements of list `l`."""
71  assert n > 0
72  (d, m) = divmod(len(l), n)
73  # If the length of `l` is divisible by `n`, use that that divisor (`d`) as size of each sublist;
74  # otherwise, the next integer value (`d + 1`).
75  s = d if m == 0 else d + 1
76  result = [l[i:i + s] for i in range(0, len(l), s)]
77  assert len(result) == n
78  return result
79
80# The prefix used in the Soong module name of all ART run-tests.
81ART_RUN_TEST_MODULE_NAME_PREFIX = "art-run-test-"
82
83# Number of shards used to declare ART run-tests in the sharded ART MTS test plan.
84NUM_MTS_ART_RUN_TEST_SHARDS = 1
85
86# Curated list of tests that have a custom `run` script, but that are
87# known to work fine with the default test execution strategy (i.e.
88# when ignoring their `run` script), even if not exactly as they would
89# with the original ART run-test harness.
90runnable_test_exceptions = frozenset([
91  "055-enum-performance",
92  "059-finalizer-throw",
93  "080-oom-throw",
94  "1004-checker-volatile-ref-load",
95  "133-static-invoke-super",
96  "1338-gc-no-los",
97  "151-OpenFileLimit",
98  "159-app-image-fields",
99  "160-read-barrier-stress",
100  "163-app-image-methods",
101  "165-lock-owner-proxy",
102  "168-vmstack-annotated",
103  "176-app-image-string",
104  "2232-write-metrics-to-log",
105  "304-method-tracing",
106  "628-vdex",
107  "643-checker-bogus-ic",
108  "676-proxy-jit-at-first-use",
109  "677-fsi2",
110  "678-quickening",
111  "818-clinit-nterp",
112  "821-madvise-willneed",
113])
114
115known_slow_tests = frozenset([
116  "175-alloc-big-bignums",
117])
118
119# Known failing ART run-tests.
120# TODO(rpl): Investigate and address the causes of failures.
121known_failing_tests = frozenset([
122  "004-SignalTest",
123  "004-UnsafeTest",
124  "051-thread",
125  "086-null-super",
126  "087-gc-after-link",
127  # 1002-notify-startup: Dependency on `libarttest` + custom `check` script.
128  "1002-notify-startup",
129  "1337-gc-coverage",
130  "1339-dead-reference-safe",
131  "136-daemon-jni-shutdown",
132  "139-register-natives",
133  "148-multithread-gc-annotations",
134  "149-suspend-all-stress",
135  "150-loadlibrary",
136  "154-gc-loop",
137  "169-threadgroup-jni",
138  "177-visibly-initialized-deadlock",
139  "179-nonvirtual-jni",
140  "1945-proxy-method-arguments",
141  "2011-stack-walk-concurrent-instrument",
142  # 2040-huge-native-alloc: Fails with:
143  #
144  #   Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0
145  #           at Main.main(Main.java:56)
146  #
147  "2040-huge-native-alloc",
148  "203-multi-checkpoint",
149  "2033-shutdown-mechanics",
150  "2036-jni-filechannel",
151  "2037-thread-name-inherit",
152  "2235-JdkUnsafeTest",
153  "305-other-fault-handler",
154  # 449-checker-bce: Dependency on `libarttest`.
155  "449-checker-bce",
156  "454-get-vreg",
157  "461-get-reference-vreg",
158  "466-get-live-vreg",
159  "497-inlining-and-class-loader",
160  "530-regression-lse",
161  "555-UnsafeGetLong-regression",
162  # 596-monitor-inflation: Dependency on `libarttest`.
163  "596-monitor-inflation",
164  "602-deoptimizeable",
165  "604-hot-static-interface",
166  "616-cha-native",
167  "616-cha-regression-proxy-method",
168  # 623-checker-loop-regressions: Dependency on `libarttest`.
169  "623-checker-loop-regressions",
170  "626-set-resolved-string",
171  "642-fp-callees",
172  "647-jni-get-field-id",
173  "655-jit-clinit",
174  "656-loop-deopt",
175  "664-aget-verifier",
176  # 680-checker-deopt-dex-pc-0: Dependency on `libarttest`.
177  "680-checker-deopt-dex-pc-0",
178  "685-deoptimizeable",
179  "687-deopt",
180  "693-vdex-inmem-loader-evict",
181  "708-jit-cache-churn",
182  # 716-jli-jit-samples: Dependency on `libarttest`.
183  "716-jli-jit-samples",
184  "717-integer-value-of",
185  "720-thread-priority",
186  # 730-cha-deopt: Fails with:
187  #
188  #   Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0
189  #           at Main.main(Main.java:24)
190  #
191  "730-cha-deopt",
192  # 813-fp-args: Dependency on `libarttest`.
193  "813-fp-args",
194  # 821-many-args: Fails with:
195  #
196  #   Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0
197  #           at Main.main(Main.java:20)
198  #
199  "821-many-args",
200  # 826-infinite-loop: The test expects an argument passed to `Main.main` (the test library,
201  # usually `arttestd` or `arttest)`, but the ART run-test TradeFed test runner
202  # (`com.android.tradefed.testtype.ArtRunTest`) does not implement this yet.
203  "826-infinite-loop",
204  # 832-cha-recursive: Dependency on `libarttest`.
205  "832-cha-recursive",
206])
207
208# ART gtests that do not need root access to the device.
209art_gtest_user_module_names = [
210    "art_libnativebridge_cts_tests",
211    "art_standalone_cmdline_tests",
212    ###"art_standalone_compiler_tests",  # b/275289981
213    # Temporarily disable this test as it is failing with ART module prebuilts (see b/243510263).
214    ### "art_standalone_dex2oat_tests",
215    ###"art_standalone_dexdump_tests",  # b/275289981
216    ###"art_standalone_dexlist_tests",  # b/275289981
217    # Temporarily disable this test as it is failing with ART module prebuilts (see b/243507635).
218    ### "art_standalone_libartbase_tests",
219    "art_standalone_libartpalette_tests",
220    "art_standalone_libartservice_tests",
221    "art_standalone_libarttools_tests",
222    "art_standalone_libdexfile_support_tests",
223    "art_standalone_libdexfile_tests",
224    "art_standalone_libprofile_tests",
225    ###"art_standalone_oatdump_tests",  # b/275289981
226    ###"art_standalone_odrefresh_tests",  # b/275289981
227    ###"art_standalone_runtime_compiler_tests",  # b/275289981
228    ###"art_standalone_runtime_tests",  # b/275289981
229    "art_standalone_sigchain_tests",
230    "libnativeloader_test",
231]
232
233# ART gtests that need root access to the device.
234art_gtest_eng_only_module_names = [
235    ###"art_standalone_dexoptanalyzer_tests",  # b/275289981
236    ###"art_standalone_profman_tests",  # b/275289981
237]
238
239# All supported ART gtests.
240art_gtest_module_names = sorted(art_gtest_user_module_names + art_gtest_eng_only_module_names)
241
242# ART gtests supported in MTS that do not need root access to the device.
243art_gtest_mts_user_module_names = copy.copy(art_gtest_user_module_names)
244# Temporarily disable `art_standalone_odrefresh_tests` in MTS,
245# as it is currently failing in Mainline testing
246# (b/206335809); a fix is in the works but may take some time
247# to land.
248#
249# TODO(b/206335809): Re-enable this test when the fix has landed.
250if "art_standalone_odrefresh_tests" in art_gtest_mts_user_module_names:
251  art_gtest_mts_user_module_names.remove("art_standalone_odrefresh_tests")
252
253# ART gtests supported in Mainline presubmits.
254art_gtests_mainline_presubmit_module_names = copy.copy(art_gtest_module_names)
255# Temporarily disable `art_standalone_odrefresh_tests` in Mainline
256# presubmits, as it is currently failing in Mainline testing
257# (b/206335809); a fix is in the works but may take some time to
258# land.
259#
260# TODO(b/206335809): Re-enable this test when the fix has landed.
261if "art_standalone_odrefresh_tests" in art_gtests_mainline_presubmit_module_names:
262  art_gtests_mainline_presubmit_module_names.remove("art_standalone_odrefresh_tests")
263
264# Tests exhibiting a flaky behavior, currently exluded from MTS for
265# the stake of stability / confidence (b/209958457).
266flaky_tests_excluded_from_mts = [
267    ("CtsLibcoreFileIOTestCases" +
268     " android.cts.FileChannelInterProcessLockTest#" + m) for m in [
269         "test_lockJJZ_Exclusive_asyncChannel",
270         "test_lockJJZ_Exclusive_syncChannel",
271         "test_lock_differentChannelTypes",
272         "test_lockJJZ_Shared_asyncChannel",
273         "test_lockJJZ_Shared_syncChannel",
274    ]
275] + [
276    ("CtsLibcoreTestCases" +
277     " com.android.org.conscrypt.javax.net.ssl.SSLSocketVersionCompatibilityTest#" + m + c)
278    for (m, c) in itertools.product(
279         [
280             "test_SSLSocket_interrupt_read_withoutAutoClose",
281             "test_SSLSocket_setSoWriteTimeout",
282         ],
283         [
284             "[0: TLSv1.2 client, TLSv1.2 server]",
285             "[1: TLSv1.2 client, TLSv1.3 server]",
286             "[2: TLSv1.3 client, TLSv1.2 server]",
287             "[3: TLSv1.3 client, TLSv1.3 server]",
288         ]
289     )
290] + [
291    ("CtsLibcoreTestCases" +
292     " libcore.dalvik.system.DelegateLastClassLoaderTest#" + m) for m in [
293         "testLookupOrderNodelegate_getResource",
294         "testLookupOrder_getResource",
295    ]
296]
297
298# Is `run_test` a Checker test (i.e. a test containing Checker
299# assertions)?
300def is_checker_test(run_test):
301  return re.match("^[0-9]+-checker-", run_test)
302
303
304class Generator:
305  def __init__(self, top_dir):
306    """Generator of ART test files for an Android source tree anchored at `top_dir`."""
307    # Path to the Android top source tree.
308    self.top_dir = top_dir
309    # Path to the ART directory
310    self.art_dir = os.path.join(top_dir, "art")
311    # Path to the ART tests directory.
312    self.art_test_dir = os.path.join(self.art_dir, "test")
313    # Path to the MTS configuration directory.
314    self.mts_config_dir = os.path.join(
315        top_dir, "test", "mts", "tools", "mts-tradefed", "res", "config")
316
317  def enumerate_run_tests(self):
318    return sorted([run_test
319                   for run_test in os.listdir(self.art_test_dir)
320                   if re.match("^[0-9]{3,}-", run_test)])
321
322  # Read build file (Bash script) and return a canonized version of it
323  # (without comments, blank lines, "debugging" statements, etc.).
324  def canonize_build_script(self, build_file):
325
326    def is_comment(line):
327      return re.match("^\\s*#", line)
328
329    def is_blank(line):
330      return re.match("^\\s*$", line)
331
332    # Is `line` a `set -e` statement?
333    def is_set_e(line):
334      return re.match("^\\s*set -e\\s*", line)
335
336    # Should `line` be kept in the canonized build script?
337    def keep_line(line):
338      return not (is_comment(line) or is_blank(line) or is_set_e(line))
339
340    with open(build_file, "r") as f:
341      lines = f.readlines()
342    return list(filter(keep_line, lines))
343
344  # Can the build script in `build_file` be safely ignored?
345  def can_ignore_build_script(self, build_file):
346    build_script = self.canonize_build_script(build_file)
347    if len(build_script) == 1:
348      if build_script[0] == "./default-build \"$@\" --experimental var-handles\n":
349        # Soong builds JARs with VarHandle support by default (i.e. by
350        # using an API level greater or equal to 28), so we can ignore
351        # build scripts that just request support for this feature.
352        return True
353    return False
354
355  # Is building `run_test` supported?
356  # TODO(b/147814778): Add build support for more tests.
357  def is_buildable(self, run_test):
358    run_test_path = os.path.join(self.art_test_dir, run_test)
359
360    # Skip tests with non-default build rules, unless these build
361    # rules can be safely ignored.
362    if os.path.isfile(os.path.join(run_test_path, "build")):
363      if not self.can_ignore_build_script(os.path.join(run_test_path, "build")):
364        return False
365    # Skip tests with sources outside the `src` directory.
366    for subdir in ["jasmin",
367                   "jasmin-multidex",
368                   "smali",
369                   "smali-ex",
370                   "smali-multidex",
371                   "src-aotex",
372                   "src-bcpex",
373                   "src-ex",
374                   "src-ex2",
375                   "src-multidex",
376                   "src2"]:
377      if os.path.isdir(os.path.join(run_test_path, subdir)):
378        return False
379    # Skip tests that have both an `src` directory and an `src-art` directory.
380    if os.path.isdir(os.path.join(run_test_path, "src")) and \
381       os.path.isdir(os.path.join(run_test_path, "src-art")):
382        return False
383    # Skip tests that have neither an `src` directory nor an `src-art` directory.
384    if not os.path.isdir(os.path.join(run_test_path, "src")) and \
385       not os.path.isdir(os.path.join(run_test_path, "src-art")):
386      return False
387    # Skip test with a copy of `sun.misc.Unsafe`.
388    if os.path.isfile(os.path.join(run_test_path, "src", "sun", "misc", "Unsafe.java")):
389      return False
390    # Skip tests with Hidden API specs.
391    if os.path.isfile(os.path.join(run_test_path, "hiddenapi-flags.csv")):
392      return False
393    # All other tests are considered buildable.
394    return True
395
396  # Is (successfully) running `run_test` supported?
397  # TODO(b/147812905): Add run-time support for more tests.
398  def is_runnable(self, run_test):
399    run_test_path = os.path.join(self.art_test_dir, run_test)
400    # Unconditionally consider some identified tests that have a
401    # (not-yet-handled) custom `run` script as runnable.
402    # TODO(rpl): Get rid of this exception mechanism by supporting
403    # these tests' `run` scripts properly.
404    if run_test in runnable_test_exceptions:
405      return True
406    # Skip tests with a custom `run` script.
407    if os.path.isfile(os.path.join(run_test_path, "run")):
408      return False
409    # Skip tests known to fail.
410    if run_test in known_failing_tests:
411      return False
412    # All other tests are considered runnable.
413    return True
414
415  def is_slow(self, run_test):
416    return run_test in known_slow_tests
417
418  def regen_bp_files(self, run_tests, buildable_tests):
419    for run_test in run_tests:
420      # Remove any previously generated file.
421      bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp")
422      if os.path.exists(bp_file):
423        logging.debug(f"Removing `{bp_file}`.")
424        os.remove(bp_file)
425
426    for run_test in buildable_tests:
427      self.regen_bp_file(run_test)
428
429  def regen_bp_file(self, run_test):
430    """Regenerate Blueprint file for an ART run-test."""
431
432    run_test_path = os.path.join(self.art_test_dir, run_test)
433    bp_file = os.path.join(run_test_path, "Android.bp")
434
435    # Optional test metadata (JSON file).
436    metadata_file = os.path.join(run_test_path, "test-metadata.json")
437    metadata = {}
438    if os.path.exists(metadata_file):
439      with open(metadata_file, "r") as f:
440        metadata = json.load(f)
441
442    run_test_module_name = ART_RUN_TEST_MODULE_NAME_PREFIX + run_test
443
444    # Set the test configuration template.
445    if self.is_runnable(run_test):
446      if "cts" in metadata.get("test_suites", []):
447        test_config_template = "art-run-test-target-cts-template"
448      elif self.is_slow(run_test):
449        test_config_template = "art-run-test-target-slow-template"
450      else:
451        test_config_template = "art-run-test-target-template"
452    else:
453      test_config_template = "art-run-test-target-no-test-suite-tag-template"
454
455    # Define `test_suites`, if present in the test's metadata.
456    test_suites = ""
457    if metadata.get("test_suites"):
458      test_suites = f"""\
459
460          test_suites: {json.dumps(metadata.get("test_suites"))},"""
461
462    if is_checker_test(run_test):
463      include_src = """\
464
465          // Include the Java source files in the test's artifacts, to make Checker assertions
466          // available to the TradeFed test runner.
467          include_srcs: true,"""
468    else:
469      include_src = ""
470
471    # The default source directory is `src`, except if `src-art` exists.
472    if os.path.isdir(os.path.join(run_test_path, "src-art")):
473      source_dir = "src-art"
474    else:
475      source_dir = "src"
476
477    with open(bp_file, "w") as f:
478      logging.debug(f"Writing `{bp_file}`.")
479      f.write(textwrap.dedent(f"""\
480      // {ADVISORY}
481
482      // Build rules for ART run-test `{run_test}`.
483
484      package {{
485          // See: http://go/android-license-faq
486          // A large-scale-change added 'default_applicable_licenses' to import
487          // all of the 'license_kinds' from "art_license"
488          // to get the below license kinds:
489          //   SPDX-license-identifier-Apache-2.0
490          default_applicable_licenses: ["art_license"],
491      }}
492
493      // Test's Dex code.
494      java_test {{
495          name: "{run_test_module_name}",
496          defaults: ["art-run-test-defaults"],
497          test_config_template: ":{test_config_template}",
498          srcs: ["{source_dir}/**/*.java"],
499          data: [
500              ":{run_test_module_name}-expected-stdout",
501              ":{run_test_module_name}-expected-stderr",
502          ],{test_suites}{include_src}
503      }}
504
505      // Test's expected standard output.
506      genrule {{
507          name: "{run_test_module_name}-expected-stdout",
508          out: ["{run_test_module_name}-expected-stdout.txt"],
509          srcs: ["expected-stdout.txt"],
510          cmd: "cp -f $(in) $(out)",
511      }}
512
513      // Test's expected standard error.
514      genrule {{
515          name: "{run_test_module_name}-expected-stderr",
516          out: ["{run_test_module_name}-expected-stderr.txt"],
517          srcs: ["expected-stderr.txt"],
518          cmd: "cp -f $(in) $(out)",
519      }}
520      """))
521
522  def regen_test_mapping_file(self, art_run_tests):
523    """Regenerate ART's `TEST_MAPPING`."""
524
525    run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests]
526
527    # Mainline presubmits.
528    mainline_other_presubmit_tests = [
529        "ComposHostTestCases",
530    ]
531    mainline_presubmit_tests = (mainline_other_presubmit_tests + run_test_module_names +
532                                art_gtests_mainline_presubmit_module_names)
533    mainline_presubmit_tests_with_apex = [t + "[com.google.android.art.apex]"
534                                          for t
535                                          in mainline_presubmit_tests]
536    mainline_presubmit_tests_dict = [{"name": t} for t in mainline_presubmit_tests_with_apex]
537
538    # Presubmits.
539    other_presubmit_tests = [
540        "CtsJdwpTestCases",
541        "BootImageProfileTest",
542        "ArtServiceTests",
543        "ComposHostTestCases",
544        "art_standalone_dexpreopt_tests",
545    ]
546    presubmit_tests = other_presubmit_tests + run_test_module_names + art_gtest_module_names
547    presubmit_tests_dict = [{"name": t} for t in presubmit_tests]
548
549    # Use an `OrderedDict` container to preserve the order in which items are inserted.
550    # Do not produce an entry for a test group if it is empty.
551    test_mapping_dict = collections.OrderedDict([
552        (test_group_name, test_group_dict)
553        for (test_group_name, test_group_dict)
554        in [
555            ("mainline-presubmit", mainline_presubmit_tests_dict),
556            ("presubmit", presubmit_tests_dict),
557        ]
558        if test_group_dict
559    ])
560    test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT)
561
562    test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING")
563    with open(test_mapping_file, "w") as f:
564      logging.debug(f"Writing `{test_mapping_file}`.")
565      f.write(f"// {ADVISORY}\n")
566      f.write(test_mapping_contents)
567      f.write("\n")
568
569  def create_mts_test_shard(self, description, tests, shard_num, copyright_year, comments = []):
570    """Factory method instantiating an `MtsTestShard`."""
571    return self.MtsTestShard(self.mts_config_dir,
572                             description, tests, shard_num, copyright_year, comments)
573
574  class MtsTestShard:
575    """Class encapsulating data and generation logic for an ART MTS test shard."""
576
577    def __init__(self, mts_config_dir, description, tests, shard_num, copyright_year, comments):
578      self.mts_config_dir = mts_config_dir
579      self.description = description
580      self.tests = tests
581      self.shard_num = shard_num
582      self.copyright_year = copyright_year
583      self.comments = comments
584
585    def shard_id(self):
586      return f"{self.shard_num:02}"
587
588    def test_plan_name(self):
589      return "mts-art-shard-" + self.shard_id()
590
591    def test_list_name(self):
592      return "mts-art-tests-list-user-shard-" + self.shard_id()
593
594    def regen_test_plan_file(self):
595      """Regenerate ART MTS test plan file shard (`mts-art-shard-<shard_num>.xml`)."""
596      root = xml.dom.minidom.Document()
597
598      advisory_header = root.createComment(f" {ADVISORY} ")
599      root.appendChild(advisory_header)
600      copyright_header = root.createComment(copyright_header_text(self.copyright_year))
601      root.appendChild(copyright_header)
602
603      configuration = root.createElement("configuration")
604      root.appendChild(configuration)
605      configuration.setAttribute(
606          "description",
607          f"Run mts-art-shard-{self.shard_id()} from a preexisting MTS installation.")
608
609      # Included XML files.
610      included_xml_files = ["mts", self.test_list_name()]
611      for xml_file in included_xml_files:
612        include = root.createElement("include")
613        include.setAttribute("name", xml_file)
614        configuration.appendChild(include)
615
616      # Test plan name.
617      option = root.createElement("option")
618      option.setAttribute("name", "plan")
619      option.setAttribute("value", self.test_plan_name())
620      configuration.appendChild(option)
621
622      xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8")
623
624      test_plan_file = os.path.join(self.mts_config_dir, self.test_plan_name() + ".xml")
625      with open(test_plan_file, "wb") as f:
626        logging.debug(f"Writing `{test_plan_file}`.")
627        f.write(xml_str)
628
629    def regen_test_list_file(self):
630      """Regenerate ART MTS test list file (`mts-art-tests-list-user-shard-<shard_num>.xml`)."""
631      root = xml.dom.minidom.Document()
632
633      advisory_header = root.createComment(f" {ADVISORY} ")
634      root.appendChild(advisory_header)
635      copyright_header = root.createComment(copyright_header_text(self.copyright_year))
636      root.appendChild(copyright_header)
637
638      configuration = root.createElement("configuration")
639      root.appendChild(configuration)
640      configuration.setAttribute(
641          "description",
642          f"List of ART MTS tests that do not need root access (shard {self.shard_id()})"
643      )
644
645      # Test declarations.
646      # ------------------
647
648      def append_test_declaration(test):
649        option = root.createElement("option")
650        option.setAttribute("name", "compatibility:include-filter")
651        option.setAttribute("value", test)
652        configuration.appendChild(option)
653
654      test_declarations_comments = [self.description + "."]
655      test_declarations_comments.extend(self.comments)
656      for c in test_declarations_comments:
657        xml_comment = root.createComment(f" {c} ")
658        configuration.appendChild(xml_comment)
659      for t in self.tests:
660        append_test_declaration(t)
661
662      # `MainlineTestModuleController` configurations.
663      # ----------------------------------------------
664
665      def append_module_controller_configuration(test):
666        option = root.createElement("option")
667        option.setAttribute("name", "compatibility:module-arg")
668        option.setAttribute("value", f"{test}:enable:true")
669        configuration.appendChild(option)
670
671      module_controller_configuration_comments = [
672          f"Enable MainlineTestModuleController for {self.description}."]
673      module_controller_configuration_comments.extend(self.comments)
674      for c in module_controller_configuration_comments:
675        xml_comment = root.createComment(f" {c} ")
676        configuration.appendChild(xml_comment)
677      for t in self.tests:
678        append_module_controller_configuration(t)
679
680      xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8")
681
682      test_list_file = os.path.join(self.mts_config_dir, self.test_list_name() + ".xml")
683      with open(test_list_file, "wb") as f:
684        logging.debug(f"Writing `{test_list_file}`.")
685        f.write(xml_str)
686
687  def regen_mts_art_tests_list_user_file(self, num_mts_art_run_test_shards):
688    """Regenerate ART MTS test list file (`mts-art-tests-list-user.xml`)."""
689    root = xml.dom.minidom.Document()
690
691    advisory_header = root.createComment(f" {ADVISORY} ")
692    root.appendChild(advisory_header)
693    copyright_header = root.createComment(copyright_header_text(2020))
694    root.appendChild(copyright_header)
695
696    configuration = root.createElement("configuration")
697    root.appendChild(configuration)
698    configuration.setAttribute("description", "List of ART MTS tests that do not need root access.")
699
700    # Included XML files.
701    for s in range(num_mts_art_run_test_shards):
702      include = root.createElement("include")
703      include.setAttribute("name", f"mts-art-tests-list-user-shard-{s:02}")
704      configuration.appendChild(include)
705
706    # Excluded flaky tests.
707    xml_comment = root.createComment(f" Excluded flaky tests (b/209958457). ")
708    configuration.appendChild(xml_comment)
709
710    def append_test_exclusion(test):
711      option = root.createElement("option")
712      option.setAttribute("name", "compatibility:exclude-filter")
713      option.setAttribute("value", test)
714      configuration.appendChild(option)
715
716    for t in flaky_tests_excluded_from_mts:
717      append_test_exclusion(t)
718
719    xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8")
720
721    mts_art_tests_list_user_file = os.path.join(self.mts_config_dir, "mts-art-tests-list-user.xml")
722    with open(mts_art_tests_list_user_file, "wb") as f:
723      logging.debug(f"Writing `{mts_art_tests_list_user_file}`.")
724      f.write(xml_str)
725
726  def regen_art_mts_files(self, art_run_tests):
727    """Regenerate ART MTS definition files."""
728
729    # Remove any previously MTS ART test plan shard (`mts-art-shard-[0-9]+.xml`)
730    # and any test list shard (`mts-art-tests-list-user-shard-[0-9]+.xml`).
731    old_test_plan_shards = sorted([
732        test_plan_shard
733        for test_plan_shard in os.listdir(self.mts_config_dir)
734        if re.match("^mts-art-(tests-list-user-)?shard-[0-9]+.xml$", test_plan_shard)])
735    for shard in old_test_plan_shards:
736      shard_path = os.path.join(self.mts_config_dir, shard)
737      if os.path.exists(shard_path):
738        logging.debug(f"Removing `{shard_path}`.")
739        os.remove(shard_path)
740
741    mts_test_shards = []
742
743    # ART test (gtest & run-test) shard(s).
744    # TODO: Also handle the case of gtests requiring root access to the device
745    # (`art_gtest_eng_only_module_names`).
746    art_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests]
747    art_run_test_shards = split_list(art_run_test_module_names, NUM_MTS_ART_RUN_TEST_SHARDS)
748    for i in range(len(art_run_test_shards)):
749      art_tests_shard_i_tests = art_run_test_shards[i]
750      # Append ART gtests to the last ART run-test shard for now.
751      # If needed, consider moving them to their own shard to increase
752      # the parallelization of code coverage runs.
753      if i + 1 == len(art_run_test_shards):
754        art_tests_shard_i_tests.extend(art_gtest_mts_user_module_names)
755      art_tests_shard_i = self.create_mts_test_shard(
756          "ART run-tests", art_tests_shard_i_tests, i, 2020,
757          ["TODO(rpl): Find a way to express this list in a more concise fashion."])
758      mts_test_shards.append(art_tests_shard_i)
759
760    # CTS Libcore non-OJ tests (`CtsLibcoreTestCases`) shard.
761    cts_libcore_tests_shard_num = len(mts_test_shards)
762    cts_libcore_tests_shard = self.create_mts_test_shard(
763        "CTS Libcore non-OJ tests", ["CtsLibcoreTestCases"], cts_libcore_tests_shard_num, 2020)
764    mts_test_shards.append(cts_libcore_tests_shard)
765
766    # Other CTS Libcore tests shard.
767    other_cts_libcore_tests_shard_num = len(mts_test_shards)
768    other_cts_libcore_tests_shard_tests = [
769        "CtsLibcoreApiEvolutionTestCases",
770        "CtsLibcoreFileIOTestCases",
771        "CtsLibcoreJsr166TestCases",
772        "CtsLibcoreLegacy22TestCases",
773        "CtsLibcoreOjTestCases",
774        "CtsLibcoreWycheproofBCTestCases",
775        "MtsLibcoreOkHttpTestCases",
776    ]
777    other_cts_libcore_tests_shard = self.create_mts_test_shard(
778        "CTS Libcore OJ tests", other_cts_libcore_tests_shard_tests,
779        other_cts_libcore_tests_shard_num, 2021)
780    mts_test_shards.append(other_cts_libcore_tests_shard)
781
782    for s in mts_test_shards:
783      s.regen_test_plan_file()
784      s.regen_test_list_file()
785
786    self.regen_mts_art_tests_list_user_file(len(mts_test_shards))
787
788  def regen_test_files(self, regen_art_mts):
789    """Regenerate ART test files.
790
791    Args:
792      regen_art_mts: If true, also regenerate the ART MTS definition.
793    """
794    run_tests = self.enumerate_run_tests()
795
796    # Create a list of the tests that can currently be built, and for
797    # which a Blueprint file is to be generated.
798    buildable_tests = list(filter(self.is_buildable, run_tests))
799
800    # Create a list of the tests that can be built and run
801    # (successfully). These tests are to be added to ART's
802    # `TEST_MAPPING` file and also tagged as part of TradeFed's
803    # `art-target-run-test` test suite via the `test-suite-tag` option
804    # in their configuration file.
805    expected_succeeding_tests = list(filter(self.is_runnable, buildable_tests))
806
807    # Regenerate Blueprint files.
808    # ---------------------------
809
810    self.regen_bp_files(run_tests, buildable_tests)
811
812    buildable_tests_percentage = int(len(buildable_tests) * 100 / len(run_tests))
813
814    print(f"Generated Blueprint files for {len(buildable_tests)} ART run-tests out of"
815          f" {len(run_tests)} ({buildable_tests_percentage}%).")
816
817    # Regenerate `TEST_MAPPING` file.
818    # -------------------------------
819
820    # Note: We only include ART run-tests expected to succeed for now.
821
822    num_presubmit_run_tests = len(expected_succeeding_tests)
823    num_mainline_presubmit_run_tests = len(expected_succeeding_tests)
824
825    self.regen_test_mapping_file(expected_succeeding_tests)
826
827    expected_succeeding_tests_percentage = int(
828        len(expected_succeeding_tests) * 100 / len(run_tests))
829
830    mainline_presubmit_gtests_percentage = int(
831        len(art_gtests_mainline_presubmit_module_names) * 100 / len(art_gtest_module_names))
832
833    print(f"Generated TEST_MAPPING entries for {len(expected_succeeding_tests)} ART run-tests out"
834          f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%):")
835    for (num_tests, test_kind, tests_percentage, test_group_name) in [
836        (num_mainline_presubmit_run_tests, "ART run-tests", 100, "mainline-presubmit"),
837        (len(art_gtests_mainline_presubmit_module_names), "ART gtests",
838         mainline_presubmit_gtests_percentage, "mainline-presubmit"),
839        (num_presubmit_run_tests, "ART run-tests", 100, "presubmit"),
840        (len(art_gtest_module_names), "ART gtests", 100, "presubmit"),
841    ]:
842      print(
843          f"  {num_tests:3d} {test_kind} ({tests_percentage}%) in `{test_group_name}` test group.")
844
845    # Regenerate ART MTS definition (optional).
846    # -----------------------------------------
847
848    if regen_art_mts:
849      self.regen_art_mts_files(expected_succeeding_tests)
850      print(f"Generated ART MTS entries for {len(expected_succeeding_tests)} ART run-tests out"
851            f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%).")
852
853def main():
854  if "ANDROID_BUILD_TOP" not in os.environ:
855    logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?")
856    sys.exit(1)
857
858  parser = argparse.ArgumentParser(
859      formatter_class=argparse.RawDescriptionHelpFormatter,
860      description=textwrap.dedent("Regenerate some ART test related files."),
861      epilog=textwrap.dedent("""\
862        Regenerate ART run-tests Blueprint files, ART's `TEST_MAPPING` file, and
863        optionally the ART MTS (Mainline Test Suite) definition.
864        """))
865  parser.add_argument("-m", "--regen-art-mts", help="regenerate the ART MTS definition as well",
866                      action="store_true")
867  parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true")
868  args = parser.parse_args()
869
870  if args.verbose:
871    logging.getLogger().setLevel(logging.DEBUG)
872
873  generator = Generator(os.path.join(os.environ["ANDROID_BUILD_TOP"]))
874  generator.regen_test_files(args.regen_art_mts)
875
876
877if __name__ == "__main__":
878  main()
879