• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18This scripts compiles Java files which are needed to execute run-tests.
19It is intended to be used only from soong genrule.
20"""
21
22import functools
23import json
24import os
25import pathlib
26import re
27import subprocess
28import sys
29import zipfile
30
31from argparse import ArgumentParser
32from concurrent.futures import ThreadPoolExecutor
33from fcntl import lockf, LOCK_EX, LOCK_NB
34from importlib.machinery import SourceFileLoader
35from os import environ, getcwd, cpu_count
36from os.path import relpath
37from pathlib import Path
38from pprint import pprint
39from shutil import copytree, rmtree
40from subprocess import PIPE, run
41from tempfile import TemporaryDirectory, NamedTemporaryFile
42from typing import Dict, List, Union, Set, Optional
43from multiprocessing import cpu_count
44
45from globals import BOOTCLASSPATH
46
47USE_RBE = 100  # Percentage of tests that can use RBE (between 0 and 100)
48
49lock_file = None  # Keep alive as long as this process is alive.
50
51RBE_COMPARE = False  # Debugging: Check that RBE and local output are identical.
52
53RBE_D8_DISABLED_FOR = {
54  "952-invoke-custom",        # b/228312861: RBE uses wrong inputs.
55  "979-const-method-handle",  # b/228312861: RBE uses wrong inputs.
56}
57
58# Debug option. Report commands that are taking a lot of user CPU time.
59REPORT_SLOW_COMMANDS = False
60
61class BuildTestContext:
62  def __init__(self, args, android_build_top, test_dir):
63    self.android_build_top = android_build_top.absolute()
64    self.bootclasspath = args.bootclasspath.absolute()
65    self.test_name = test_dir.name
66    self.test_dir = test_dir.absolute()
67    self.mode = args.mode
68    self.jvm = (self.mode == "jvm")
69    self.host = (self.mode == "host")
70    self.target = (self.mode == "target")
71    assert self.jvm or self.host or self.target
72
73    self.java_home = Path(os.environ.get("JAVA_HOME")).absolute()
74    self.java_path = self.java_home / "bin/java"
75    self.javac_path = self.java_home / "bin/javac"
76    self.javac_args = "-g -Xlint:-options"
77
78    # Helper functions to execute tools.
79    self.d8_path = args.d8.absolute()
80    self.d8 = functools.partial(self.run, args.d8.absolute())
81    self.jasmin = functools.partial(self.run, args.jasmin.absolute())
82    self.javac = functools.partial(self.run, self.javac_path)
83    self.smali_path = args.smali.absolute()
84    self.rbe_rewrapper = args.rewrapper.absolute()
85    self.smali = functools.partial(self.run, args.smali.absolute())
86    self.soong_zip = functools.partial(self.run, args.soong_zip.absolute())
87    self.zipalign = functools.partial(self.run, args.zipalign.absolute())
88    if args.hiddenapi:
89      self.hiddenapi = functools.partial(self.run, args.hiddenapi.absolute())
90
91    # RBE wrapper for some of the tools.
92    if "RBE_server_address" in os.environ and USE_RBE > (hash(self.test_name) % 100):
93      self.rbe_exec_root = os.environ.get("RBE_exec_root")
94
95      # TODO(b/307932183) Regression: RBE produces wrong output for D8 in ART
96      disable_d8 = any((self.test_dir / n).exists() for n in ["classes", "src2", "src-art"])
97
98      if self.test_name not in RBE_D8_DISABLED_FOR and not disable_d8:
99        self.d8 = functools.partial(self.rbe_d8, args.d8.absolute())
100      self.javac = functools.partial(self.rbe_javac, self.javac_path)
101      self.smali = functools.partial(self.rbe_smali, args.smali.absolute())
102
103    # Minimal environment needed for bash commands that we execute.
104    self.bash_env = {
105      "ANDROID_BUILD_TOP": self.android_build_top,
106      "D8": args.d8.absolute(),
107      "JAVA": self.java_path,
108      "JAVAC": self.javac_path,
109      "JAVAC_ARGS": self.javac_args,
110      "JAVA_HOME": self.java_home,
111      "PATH": os.environ["PATH"],
112      "PYTHONDONTWRITEBYTECODE": "1",
113      "SMALI": args.smali.absolute(),
114      "SOONG_ZIP": args.soong_zip.absolute(),
115      "TEST_NAME": self.test_name,
116    }
117
118  def bash(self, cmd):
119    return subprocess.run(cmd,
120                          shell=True,
121                          cwd=self.test_dir,
122                          env=self.bash_env,
123                          check=True)
124
125  def run(self, executable: pathlib.Path, args: List[Union[pathlib.Path, str]]):
126    assert isinstance(executable, pathlib.Path), executable
127    cmd: List[Union[pathlib.Path, str]] = []
128    if REPORT_SLOW_COMMANDS:
129      cmd += ["/usr/bin/time"]
130    if executable.suffix == ".sh":
131      cmd += ["/bin/bash"]
132    cmd += [executable]
133    cmd += args
134    env = self.bash_env
135    env.update({k: v for k, v in os.environ.items() if k.startswith("RBE_")})
136    # Make paths relative as otherwise we could create too long command line.
137    for i, arg in enumerate(cmd):
138      if isinstance(arg, pathlib.Path):
139        assert arg.absolute(), arg
140        cmd[i] = relpath(arg, self.test_dir)
141      elif isinstance(arg, list):
142        assert all(p.absolute() for p in arg), arg
143        cmd[i] = ":".join(relpath(p, self.test_dir) for p in arg)
144      else:
145        assert isinstance(arg, str), arg
146    p = subprocess.run(cmd,
147                       encoding=sys.stdout.encoding,
148                       cwd=self.test_dir,
149                       env=self.bash_env,
150                       stderr=subprocess.STDOUT,
151                       stdout=subprocess.PIPE)
152    if REPORT_SLOW_COMMANDS:
153      m = re.search(r"([0-9\.]+)user", p.stdout)
154      assert m, p.stdout
155      t = float(m.group(1))
156      if t > 1.0:
157        cmd_text = " ".join(map(str, cmd[1:]))[:100]
158        print(f"[{self.test_name}] Command took {t:.2f}s: {cmd_text}")
159
160    if p.returncode != 0:
161      raise Exception("Command failed with exit code {}\n$ {}\n{}".format(
162                      p.returncode, " ".join(map(str, cmd)), p.stdout))
163    return p
164
165  def rbe_wrap(self, args, inputs: Set[pathlib.Path]=None):
166    with NamedTemporaryFile(mode="w+t") as input_list:
167      inputs = inputs or set()
168      for i in inputs:
169        assert i.exists(), i
170      for i, arg in enumerate(args):
171        if isinstance(arg, pathlib.Path):
172          assert arg.absolute(), arg
173          inputs.add(arg)
174        elif isinstance(arg, list):
175          assert all(p.absolute() for p in arg), arg
176          inputs.update(arg)
177      input_list.writelines([relpath(i, self.rbe_exec_root)+"\n" for i in inputs])
178      input_list.flush()
179      dbg_args = ["-compare", "-num_local_reruns=1", "-num_remote_reruns=1"] if RBE_COMPARE else []
180      return self.run(self.rbe_rewrapper, [
181        "--platform=" + os.environ["RBE_platform"],
182        "--input_list_paths=" + input_list.name,
183      ] + dbg_args + args)
184
185  def rbe_javac(self, javac_path:Path, args):
186    output = relpath(Path(args[args.index("-d") + 1]), self.rbe_exec_root)
187    return self.rbe_wrap(["--output_directories", output, javac_path] + args)
188
189  def rbe_d8(self, d8_path:Path, args):
190    inputs = set([d8_path.parent.parent / "framework/d8.jar"])
191    output = relpath(Path(args[args.index("--output") + 1]), self.rbe_exec_root)
192    return self.rbe_wrap([
193      "--output_files" if output.endswith(".jar") else "--output_directories", output,
194      "--toolchain_inputs=prebuilts/jdk/jdk21/linux-x86/bin/java",
195      d8_path] + args, inputs)
196
197  def rbe_smali(self, smali_path:Path, args):
198    # The output of smali is non-deterministic, so create wrapper script,
199    # which runs D8 on the output to normalize it.
200    api = args[args.index("--api") + 1]
201    output = Path(args[args.index("--output") + 1])
202    wrapper = output.with_suffix(".sh")
203    wrapper.write_text('''
204      set -e
205      {smali} $@
206      mkdir dex_normalize
207      {d8} --min-api {api} --output dex_normalize {output}
208      cp dex_normalize/classes.dex {output}
209      rm -rf dex_normalize
210    '''.strip().format(
211      smali=relpath(self.smali_path, self.test_dir),
212      d8=relpath(self.d8_path, self.test_dir),
213      api=api,
214      output=relpath(output, self.test_dir),
215    ))
216
217    inputs = set([
218      wrapper,
219      self.smali_path,
220      self.smali_path.parent.parent / "framework/android-smali.jar",
221      self.d8_path,
222      self.d8_path.parent.parent / "framework/d8.jar",
223    ])
224    res = self.rbe_wrap([
225      "--output_files", relpath(output, self.rbe_exec_root),
226      "--toolchain_inputs=prebuilts/jdk/jdk21/linux-x86/bin/java",
227      "/bin/bash", wrapper] + args, inputs)
228    wrapper.unlink()
229    return res
230
231  def build(self) -> None:
232    script = self.test_dir / "build.py"
233    if script.exists():
234      module = SourceFileLoader("build_" + self.test_name,
235                                str(script)).load_module()
236      module.build(self)
237    else:
238      self.default_build()
239
240  def default_build(
241      self,
242      use_desugar=True,
243      use_hiddenapi=True,
244      need_dex=None,
245      zip_compression_method="deflate",
246      zip_align_bytes=None,
247      api_level:Union[int, str]=26,  # Can also be named alias (string).
248      javac_args=[],
249      javac_classpath: List[Path]=[],
250      d8_flags=[],
251      d8_dex_container=True,
252      smali_args=[],
253      use_smali=True,
254      use_jasmin=True,
255      javac_source_arg="1.8",
256      javac_target_arg="1.8",
257      delete_srcs=True,
258    ):
259    javac_classpath = javac_classpath.copy()  # Do not modify default value.
260
261    # Wrap "pathlib.Path" with our own version that ensures all paths are absolute.
262    # Plain filenames are assumed to be relative to self.test_dir and made absolute.
263    class Path(pathlib.Path):
264      def __new__(cls, filename: str):
265        path = pathlib.Path(filename)
266        return path if path.is_absolute() else (self.test_dir / path)
267
268    need_dex = (self.host or self.target) if need_dex is None else need_dex
269
270    if self.jvm:
271      # No desugaring on jvm because it supports the latest functionality.
272      use_desugar = False
273
274    # Set API level for smali and d8.
275    if isinstance(api_level, str):
276      API_LEVEL = {
277        "default-methods": 24,
278        "parameter-annotations": 25,
279        "agents": 26,
280        "method-handles": 26,
281        "var-handles": 28,
282        "const-method-type": 28,
283      }
284      api_level = API_LEVEL[api_level]
285    assert isinstance(api_level, int), api_level
286
287    def zip(zip_target: Path, *files: Path):
288      zip_args = ["-o", zip_target, "-C", zip_target.parent]
289      if zip_compression_method == "store":
290        zip_args.extend(["-L", "0"])
291      for f in files:
292        zip_args.extend(["-f", f])
293      self.soong_zip(zip_args)
294
295      if zip_align_bytes:
296        # zipalign does not operate in-place, so write results to a temp file.
297        with TemporaryDirectory() as tmp_dir:
298          tmp_file = Path(tmp_dir) / "aligned.zip"
299          self.zipalign(["-f", str(zip_align_bytes), zip_target, tmp_file])
300          # replace original zip target with our temp file.
301          tmp_file.rename(zip_target)
302
303
304    def make_jasmin(dst_dir: Path, src_dir: Path) -> Optional[Path]:
305      if not use_jasmin or not src_dir.exists():
306        return None  # No sources to compile.
307      dst_dir.mkdir()
308      self.jasmin(["-d", dst_dir] + sorted(src_dir.glob("**/*.j")))
309      return dst_dir
310
311    def make_smali(dst_dex: Path, src_dir: Path) -> Optional[Path]:
312      if not use_smali or not src_dir.exists():
313        return None  # No sources to compile.
314      p = self.smali(["-JXmx512m", "assemble"] + smali_args + ["--api", str(api_level)] +
315                     ["--output", dst_dex] + sorted(src_dir.glob("**/*.smali")))
316      assert dst_dex.exists(), p.stdout  # NB: smali returns 0 exit code even on failure.
317      return dst_dex
318
319    def make_java(dst_dir: Path, *src_dirs: Path) -> Optional[Path]:
320      if not any(src_dir.exists() for src_dir in src_dirs):
321        return None  # No sources to compile.
322      dst_dir.mkdir(exist_ok=True)
323      args = self.javac_args.split(" ") + javac_args
324      args += ["-implicit:none", "-encoding", "utf8", "-d", dst_dir]
325      args += ["-source", javac_source_arg, "-target", javac_target_arg]
326      if not self.jvm and float(javac_target_arg) < 17.0:
327        args += ["-bootclasspath", self.bootclasspath]
328      if javac_classpath:
329        args += ["-classpath", javac_classpath]
330      for src_dir in src_dirs:
331        args += sorted(src_dir.glob("**/*.java"))
332      self.javac(args)
333      javac_post = Path("javac_post.sh")
334      if javac_post.exists():
335        self.run(javac_post, [dst_dir])
336      return dst_dir
337
338
339    # Make a "dex" file given a directory of classes. This will be
340    # packaged in a jar file.
341    def make_dex(src_dir: Path):
342      dst_jar = Path(src_dir.name + ".jar")
343      args = []
344      if d8_dex_container:
345        args += ["-JDcom.android.tools.r8.dexContainerExperiment"]
346      args += d8_flags + ["--min-api", str(api_level), "--output", dst_jar]
347      args += ["--lib", self.bootclasspath] if use_desugar else ["--no-desugaring"]
348      args += sorted(src_dir.glob("**/*.class"))
349      self.d8(args)
350
351      # D8 outputs to JAR files today rather than DEX files as DX used
352      # to. To compensate, we extract the DEX from d8's output to meet the
353      # expectations of make_dex callers.
354      dst_dex = Path(src_dir.name + ".dex")
355      with TemporaryDirectory() as tmp_dir:
356        zipfile.ZipFile(dst_jar, "r").extractall(tmp_dir)
357        (Path(tmp_dir) / "classes.dex").rename(dst_dex)
358
359    # Merge all the dex files.
360    # Skip non-existing files, but at least 1 file must exist.
361    def make_dexmerge(dst_dex: Path, *src_dexs: Path):
362      # Include destination. Skip any non-existing files.
363      srcs = [f for f in [dst_dex] + list(src_dexs) if f.exists()]
364
365      # NB: We merge even if there is just single input.
366      # It is useful to normalize non-deterministic smali output.
367      tmp_dir = self.test_dir / "dexmerge"
368      tmp_dir.mkdir()
369      flags = []
370      if d8_dex_container:
371        flags += ["-JDcom.android.tools.r8.dexContainerExperiment"]
372      flags += ["--min-api", str(api_level), "--output", tmp_dir]
373      self.d8(flags + srcs)
374      assert not (tmp_dir / "classes2.dex").exists()
375      for src_file in srcs:
376        src_file.unlink()
377      (tmp_dir / "classes.dex").rename(dst_dex)
378      tmp_dir.rmdir()
379
380
381    def make_hiddenapi(*dex_files: Path):
382      if not use_hiddenapi or not Path("hiddenapi-flags.csv").exists():
383        return  # Nothing to do.
384      args: List[Union[str, Path]] = ["encode"]
385      for dex_file in dex_files:
386        args.extend(["--input-dex=" + str(dex_file), "--output-dex=" + str(dex_file)])
387      args.append("--api-flags=hiddenapi-flags.csv")
388      args.append("--no-force-assign-all")
389      self.hiddenapi(args)
390
391
392    if Path("classes.dex").exists():
393      zip(Path(self.test_name + ".jar"), Path("classes.dex"))
394      return
395
396    if Path("classes.dm").exists():
397      zip(Path(self.test_name + ".jar"), Path("classes.dm"))
398      return
399
400    if make_jasmin(Path("jasmin_classes"), Path("jasmin")):
401      javac_classpath.append(Path("jasmin_classes"))
402
403    if make_jasmin(Path("jasmin_classes2"), Path("jasmin-multidex")):
404      javac_classpath.append(Path("jasmin_classes2"))
405
406    # To allow circular references, compile src/, src-multidex/, src-aotex/,
407    # src-bcpex/, src-ex/ together and pass the output as class path argument.
408    # Replacement sources in src-art/, src2/ and src-ex2/ can replace symbols
409    # used by the other src-* sources we compile here but everything needed to
410    # compile the other src-* sources should be present in src/ (and jasmin*/).
411    extra_srcs = ["src-multidex", "src-aotex", "src-bcpex", "src-ex"]
412    replacement_srcs = ["src2", "src-ex2"] + ([] if self.jvm else ["src-art"])
413    if (Path("src").exists() and
414        any(Path(p).exists() for p in extra_srcs + replacement_srcs)):
415      make_java(Path("classes-tmp-all"), Path("src"), *map(Path, extra_srcs))
416      javac_classpath.append(Path("classes-tmp-all"))
417
418    if make_java(Path("classes-aotex"), Path("src-aotex")) and need_dex:
419      make_dex(Path("classes-aotex"))
420      # rename it so it shows up as "classes.dex" in the zip file.
421      Path("classes-aotex.dex").rename(Path("classes.dex"))
422      zip(Path(self.test_name + "-aotex.jar"), Path("classes.dex"))
423
424    if make_java(Path("classes-bcpex"), Path("src-bcpex")) and need_dex:
425      make_dex(Path("classes-bcpex"))
426      # rename it so it shows up as "classes.dex" in the zip file.
427      Path("classes-bcpex.dex").rename(Path("classes.dex"))
428      zip(Path(self.test_name + "-bcpex.jar"), Path("classes.dex"))
429
430    make_java(Path("classes"), Path("src"))
431
432    if not self.jvm:
433      # Do not attempt to build src-art directories on jvm,
434      # since it would fail without libcore.
435      make_java(Path("classes"), Path("src-art"))
436
437    if make_java(Path("classes2"), Path("src-multidex")) and need_dex:
438      make_dex(Path("classes2"))
439
440    make_java(Path("classes"), Path("src2"))
441
442    # If the classes directory is not-empty, package classes in a DEX file.
443    # NB: some tests provide classes rather than java files.
444    if any(Path("classes").glob("*")) and need_dex:
445      make_dex(Path("classes"))
446
447    if Path("jasmin_classes").exists():
448      # Compile Jasmin classes as if they were part of the classes.dex file.
449      if need_dex:
450        make_dex(Path("jasmin_classes"))
451        make_dexmerge(Path("classes.dex"), Path("jasmin_classes.dex"))
452      else:
453        # Move jasmin classes into classes directory so that they are picked up
454        # with -cp classes.
455        Path("classes").mkdir(exist_ok=True)
456        copytree(Path("jasmin_classes"), Path("classes"), dirs_exist_ok=True)
457
458    if need_dex and make_smali(Path("smali_classes.dex"), Path("smali")):
459      # Merge smali files into classes.dex,
460      # this takes priority over any jasmin files.
461      make_dexmerge(Path("classes.dex"), Path("smali_classes.dex"))
462
463    # Compile Jasmin classes in jasmin-multidex as if they were part of
464    # the classes2.jar
465    if Path("jasmin-multidex").exists():
466      if need_dex:
467        make_dex(Path("jasmin_classes2"))
468        make_dexmerge(Path("classes2.dex"), Path("jasmin_classes2.dex"))
469      else:
470        # Move jasmin classes into classes2 directory so that
471        # they are picked up with -cp classes2.
472        Path("classes2").mkdir()
473        copytree(Path("jasmin_classes2"), Path("classes2"), dirs_exist_ok=True)
474        rmtree(Path("jasmin_classes2"))
475
476    if need_dex and make_smali(Path("smali_classes2.dex"), Path("smali-multidex")):
477      # Merge smali_classes2.dex into classes2.dex
478      make_dexmerge(Path("classes2.dex"), Path("smali_classes2.dex"))
479
480    make_java(Path("classes-ex"), Path("src-ex"))
481
482    make_java(Path("classes-ex"), Path("src-ex2"))
483
484    if Path("classes-ex").exists() and need_dex:
485      make_dex(Path("classes-ex"))
486
487    if need_dex and make_smali(Path("smali_classes-ex.dex"), Path("smali-ex")):
488      # Merge smali files into classes-ex.dex.
489      make_dexmerge(Path("classes-ex.dex"), Path("smali_classes-ex.dex"))
490
491    if Path("classes-ex.dex").exists():
492      # Apply hiddenapi on the dex files if the test has API list file(s).
493      make_hiddenapi(Path("classes-ex.dex"))
494
495      # quick shuffle so that the stored name is "classes.dex"
496      Path("classes.dex").rename(Path("classes-1.dex"))
497      Path("classes-ex.dex").rename(Path("classes.dex"))
498      zip(Path(self.test_name + "-ex.jar"), Path("classes.dex"))
499      Path("classes.dex").rename(Path("classes-ex.dex"))
500      Path("classes-1.dex").rename(Path("classes.dex"))
501
502    # Apply hiddenapi on the dex files if the test has API list file(s).
503    if need_dex:
504      if any(Path(".").glob("*-multidex")):
505        make_hiddenapi(Path("classes.dex"), Path("classes2.dex"))
506      else:
507        make_hiddenapi(Path("classes.dex"))
508
509    # Clean up intermediate files.
510    if self.target or self.host:
511      for f in self.test_dir.glob("**/*.class"):
512        f.unlink()
513    if delete_srcs and "-checker-" not in self.test_name:
514      for ext in ["java", "smali", "j"]:
515        for f in self.test_dir.glob(f"**/*.{ext}"):
516          f.unlink()
517
518    # Create a single dex jar with two dex files for multidex.
519    if need_dex:
520      if Path("classes2.dex").exists():
521        zip(Path(self.test_name + ".jar"), Path("classes.dex"), Path("classes2.dex"))
522      else:
523        zip(Path(self.test_name + ".jar"), Path("classes.dex"))
524
525# Create bash script that compiles the boot image on device.
526# This is currently only used for eng-prod testing (which is different
527# to the local and LUCI code paths that use buildbot-sync.sh script).
528def create_setup_script(is64: bool):
529  out = "/data/local/tmp/art/apex/art_boot_images"
530  isa = 'arm64' if is64 else 'arm'
531  jar = BOOTCLASSPATH
532  cmd = [
533    f"/apex/com.android.art/bin/{'dex2oat64' if is64 else 'dex2oat32'}",
534    "--runtime-arg", f"-Xbootclasspath:{':'.join(jar)}",
535    "--runtime-arg", f"-Xbootclasspath-locations:{':'.join(jar)}",
536  ] + [f"--dex-file={j}" for j in jar] + [f"--dex-location={j}" for j in jar] + [
537    f"--instruction-set={isa}",
538    "--base=0x70000000",
539    "--compiler-filter=speed-profile",
540    "--profile-file=/apex/com.android.art/etc/boot-image.prof",
541    "--avoid-storing-invocation",
542    "--generate-debug-info",
543    "--generate-build-id",
544    "--image-format=lz4hc",
545    "--strip",
546    "--android-root=out/empty",
547    f"--image={out}/{isa}/boot.art",
548    f"--oat-file={out}/{isa}/boot.oat",
549  ]
550  return [
551    f"rm -rf {out}/{isa}",
552    f"mkdir -p {out}/{isa}",
553    " ".join(cmd),
554  ]
555
556# Create bash scripts that can fully execute the run tests.
557# This can be used in CI to execute the tests without running `testrunner.py`.
558# This takes into account any custom behaviour defined in per-test `run.py`.
559# We generate distinct scripts for all of the pre-defined variants.
560def create_ci_runner_scripts(out, mode, test_names):
561  out.mkdir(parents=True)
562  setup = out / "setup.sh"
563  setup_script = create_setup_script(False) + create_setup_script(True)
564  setup.write_text("\n".join(setup_script))
565
566  python = sys.executable
567  script = 'art/test/testrunner/testrunner.py'
568  envs = {
569    "ANDROID_BUILD_TOP": str(Path(getcwd()).absolute()),
570    "ART_TEST_RUN_FROM_SOONG": "true",
571    # TODO: Make the runner scripts target agnostic.
572    #       The only dependency is setting of "-Djava.library.path".
573    "TARGET_ARCH": "arm64",
574    "TARGET_2ND_ARCH": "arm",
575    "TMPDIR": Path(getcwd()) / "tmp",
576  }
577  args = [
578    f"--run-test-option=--create-runner={out}",
579    f"-j={cpu_count()}",
580    f"--{mode}",
581  ]
582  run([python, script] + args + test_names, env=envs, check=True)
583  tests = {
584    "setup#compile-boot-image": {
585      "adb push": [
586        [str(setup.relative_to(out)), "/data/local/tmp/art/setup.sh"]
587      ],
588      "adb shell": [
589        ["rm", "-rf", "/data/local/tmp/art/test"],
590        ["sh", "/data/local/tmp/art/setup.sh"],
591      ],
592    },
593  }
594  for runner in Path(out).glob("*/*.sh"):
595    test_name = runner.parent.name
596    test_hash = runner.stem
597    target_dir = f"/data/local/tmp/art/test/{test_hash}"
598    tests[f"{test_name}#{test_hash}"] = {
599      "dependencies": ["setup#compile-boot-image"],
600      "adb push": [
601        [f"../{mode}/{test_name}", f"{target_dir}"],
602        [str(runner.relative_to(out)), f"{target_dir}/run.sh"]
603      ],
604      "adb shell": [["sh", f"{target_dir}/run.sh"]],
605    }
606  return tests
607
608# If we build just individual shard, we want to split the work among all the cores,
609# but if the build system builds all shards, we don't want to overload the machine.
610# We don't know which situation we are in, so as simple work-around, we use a lock
611# file to allow only one shard to use multiprocessing at the same time.
612def use_multiprocessing(mode: str) -> bool:
613  if "RBE_server_address" in os.environ:
614    return True
615  global lock_file
616  lock_path = Path(environ["TMPDIR"]) / ("art-test-run-test-build-py-" + mode)
617  lock_file = open(lock_path, "w")
618  try:
619    lockf(lock_file, LOCK_EX | LOCK_NB)
620    return True  # We are the only instance of this script in the build system.
621  except BlockingIOError:
622    return False  # Some other instance is already running.
623
624
625def main() -> None:
626  parser = ArgumentParser(description=__doc__)
627  parser.add_argument("--out", type=Path, help="Final zip file")
628  parser.add_argument("--mode", choices=["host", "jvm", "target"])
629  parser.add_argument("--bootclasspath", type=Path)
630  parser.add_argument("--d8", type=Path)
631  parser.add_argument("--hiddenapi", type=Path)
632  parser.add_argument("--jasmin", type=Path)
633  parser.add_argument("--rewrapper", type=Path)
634  parser.add_argument("--smali", type=Path)
635  parser.add_argument("--soong_zip", type=Path)
636  parser.add_argument("--zipalign", type=Path)
637  parser.add_argument("--test-dir-regex")
638  parser.add_argument("srcs", nargs="+", type=Path)
639  args = parser.parse_args()
640
641  android_build_top = Path(getcwd()).absolute()
642  ziproot = args.out.absolute().parent / "zip"
643  test_dir_regex = re.compile(args.test_dir_regex) if args.test_dir_regex else re.compile(".*")
644  srcdirs = set(s.parents[-4].absolute() for s in args.srcs if test_dir_regex.search(str(s)))
645
646  # Special hidden-api shard: If the --hiddenapi flag is provided, build only
647  # hiddenapi tests. Otherwise exclude all hiddenapi tests from normal shards.
648  def filter_by_hiddenapi(srcdir: Path) -> bool:
649    return (args.hiddenapi != None) == ("hiddenapi" in srcdir.name)
650
651  # Initialize the test objects.
652  # We need to do this before we change the working directory below.
653  tests: List[BuildTestContext] = []
654  for srcdir in filter(filter_by_hiddenapi, srcdirs):
655    dstdir = ziproot / args.mode / srcdir.name
656    copytree(srcdir, dstdir)
657    tests.append(BuildTestContext(args, android_build_top, dstdir))
658
659  # We can not change the working directory per each thread since they all run in parallel.
660  # Create invalid read-only directory to catch accidental use of current working directory.
661  with TemporaryDirectory("-do-not-use-cwd") as invalid_tmpdir:
662    os.chdir(invalid_tmpdir)
663    os.chmod(invalid_tmpdir, 0)
664    with ThreadPoolExecutor(cpu_count() if use_multiprocessing(args.mode) else 1) as pool:
665      jobs = {ctx.test_name: pool.submit(ctx.build) for ctx in tests}
666      for test_name, job in jobs.items():
667        try:
668          job.result()
669        except Exception as e:
670          raise Exception("Failed to build " + test_name) from e
671
672  if args.mode == "target":
673    os.chdir(android_build_top)
674    test_names = [ctx.test_name for ctx in tests]
675    dst = ziproot / "runner" / args.out.with_suffix(".tests.json").name
676    tests = create_ci_runner_scripts(dst.parent, args.mode, test_names)
677    dst.write_text(json.dumps(tests, indent=2, sort_keys=True))
678
679  # Create the final zip file which contains the content of the temporary directory.
680  soong_zip = android_build_top / args.soong_zip
681  zip_file = android_build_top / args.out
682  run([soong_zip, "-L", "0", "-o", zip_file, "-C", ziproot, "-D", ziproot], check=True)
683
684if __name__ == "__main__":
685  main()
686