1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18This scripts compiles Java files which are needed to execute run-tests. 19It is intended to be used only from soong genrule. 20""" 21 22import argparse 23import functools 24import glob 25import os 26import pathlib 27import shlex 28import shutil 29import subprocess 30import sys 31import zipfile 32 33from argparse import ArgumentParser 34from fcntl import lockf, LOCK_EX, LOCK_NB 35from importlib.machinery import SourceFileLoader 36from concurrent.futures import ThreadPoolExecutor 37from os import environ, getcwd, chdir, cpu_count, chmod 38from os.path import relpath 39from pathlib import Path 40from pprint import pprint 41from re import match 42from shutil import copytree, rmtree 43from subprocess import run 44from tempfile import TemporaryDirectory, NamedTemporaryFile 45from typing import Dict, List, Union, Set, Optional 46 47USE_RBE = 100 # Percentage of tests that can use RBE (between 0 and 100) 48 49lock_file = None # Keep alive as long as this process is alive. 50 51RBE_D8_DISABLED_FOR = { 52 "952-invoke-custom", # b/228312861: RBE uses wrong inputs. 53 "979-const-method-handle", # b/228312861: RBE uses wrong inputs. 54} 55 56class BuildTestContext: 57 def __init__(self, args, android_build_top, test_dir): 58 self.android_build_top = android_build_top.absolute() 59 self.bootclasspath = args.bootclasspath.absolute() 60 self.test_name = test_dir.name 61 self.test_dir = test_dir.absolute() 62 self.mode = args.mode 63 self.jvm = (self.mode == "jvm") 64 self.host = (self.mode == "host") 65 self.target = (self.mode == "target") 66 assert self.jvm or self.host or self.target 67 68 self.java_home = Path(os.environ.get("JAVA_HOME")).absolute() 69 self.java_path = self.java_home / "bin/java" 70 self.javac_path = self.java_home / "bin/javac" 71 self.javac_args = "-g -Xlint:-options -source 1.8 -target 1.8" 72 73 # Helper functions to execute tools. 74 self.d8 = functools.partial(self.run, args.d8.absolute()) 75 self.jasmin = functools.partial(self.run, args.jasmin.absolute()) 76 self.javac = functools.partial(self.run, self.javac_path) 77 self.smali = functools.partial(self.run, args.smali.absolute()) 78 self.soong_zip = functools.partial(self.run, args.soong_zip.absolute()) 79 self.zipalign = functools.partial(self.run, args.zipalign.absolute()) 80 if args.hiddenapi: 81 self.hiddenapi = functools.partial(self.run, args.hiddenapi.absolute()) 82 83 # RBE wrapper for some of the tools. 84 if "RBE_server_address" in os.environ and USE_RBE > (hash(self.test_name) % 100): 85 self.rbe_exec_root = os.environ.get("RBE_exec_root") 86 self.rbe_rewrapper = self.android_build_top / "prebuilts/remoteexecution-client/live/rewrapper" 87 if self.test_name not in RBE_D8_DISABLED_FOR: 88 self.d8 = functools.partial(self.rbe_d8, args.d8.absolute()) 89 self.javac = functools.partial(self.rbe_javac, self.javac_path) 90 self.smali = functools.partial(self.rbe_smali, args.smali.absolute()) 91 92 # Minimal environment needed for bash commands that we execute. 93 self.bash_env = { 94 "ANDROID_BUILD_TOP": self.android_build_top, 95 "D8": args.d8.absolute(), 96 "JAVA": self.java_path, 97 "JAVAC": self.javac_path, 98 "JAVAC_ARGS": self.javac_args, 99 "JAVA_HOME": self.java_home, 100 "PATH": os.environ["PATH"], 101 "PYTHONDONTWRITEBYTECODE": "1", 102 "SMALI": args.smali.absolute(), 103 "SOONG_ZIP": args.soong_zip.absolute(), 104 "TEST_NAME": self.test_name, 105 } 106 107 def bash(self, cmd): 108 return subprocess.run(cmd, 109 shell=True, 110 cwd=self.test_dir, 111 env=self.bash_env, 112 check=True) 113 114 def run(self, executable: pathlib.Path, args: List[Union[pathlib.Path, str]]): 115 assert isinstance(executable, pathlib.Path), executable 116 cmd: List[Union[pathlib.Path, str]] = [] 117 if executable.suffix == ".sh": 118 cmd += ["/bin/bash"] 119 cmd += [executable] 120 cmd += args 121 env = self.bash_env 122 env.update({k: v for k, v in os.environ.items() if k.startswith("RBE_")}) 123 # Make paths relative as otherwise we could create too long command line. 124 for i, arg in enumerate(cmd): 125 if isinstance(arg, pathlib.Path): 126 assert arg.absolute(), arg 127 cmd[i] = relpath(arg, self.test_dir) 128 elif isinstance(arg, list): 129 assert all(p.absolute() for p in arg), arg 130 cmd[i] = ":".join(relpath(p, self.test_dir) for p in arg) 131 else: 132 assert isinstance(arg, str), arg 133 p = subprocess.run(cmd, 134 encoding=sys.stdout.encoding, 135 cwd=self.test_dir, 136 env=self.bash_env, 137 stderr=subprocess.STDOUT, 138 stdout=subprocess.PIPE) 139 if p.returncode != 0: 140 raise Exception("Command failed with exit code {}\n$ {}\n{}".format( 141 p.returncode, " ".join(map(str, cmd)), p.stdout)) 142 return p 143 144 def rbe_wrap(self, args, inputs: Set[pathlib.Path]=None): 145 with NamedTemporaryFile(mode="w+t") as input_list: 146 inputs = inputs or set() 147 for i, arg in enumerate(args): 148 if isinstance(arg, pathlib.Path): 149 assert arg.absolute(), arg 150 inputs.add(arg) 151 elif isinstance(arg, list): 152 assert all(p.absolute() for p in arg), arg 153 inputs.update(arg) 154 input_list.writelines([relpath(i, self.rbe_exec_root)+"\n" for i in inputs]) 155 input_list.flush() 156 return self.run(self.rbe_rewrapper, [ 157 "--platform=" + os.environ["RBE_platform"], 158 "--input_list_paths=" + input_list.name, 159 ] + args) 160 161 def rbe_javac(self, javac_path:Path, args): 162 output = relpath(Path(args[args.index("-d") + 1]), self.rbe_exec_root) 163 return self.rbe_wrap(["--output_directories", output, javac_path] + args) 164 165 def rbe_d8(self, d8_path:Path, args): 166 inputs = set([d8_path.parent.parent / "framework/d8.jar"]) 167 output = relpath(Path(args[args.index("--output") + 1]), self.rbe_exec_root) 168 return self.rbe_wrap([ 169 "--output_files" if output.endswith(".jar") else "--output_directories", output, 170 "--toolchain_inputs=prebuilts/jdk/jdk17/linux-x86/bin/java", 171 d8_path] + args, inputs) 172 173 def rbe_smali(self, smali_path:Path, args): 174 inputs = set([smali_path.parent.parent / "framework/smali.jar"]) 175 output = relpath(Path(args[args.index("--output") + 1]), self.rbe_exec_root) 176 return self.rbe_wrap([ 177 "--output_files", output, 178 "--toolchain_inputs=prebuilts/jdk/jdk17/linux-x86/bin/java", 179 smali_path] + args, inputs) 180 181 def build(self) -> None: 182 script = self.test_dir / "build.py" 183 if script.exists(): 184 module = SourceFileLoader("build_" + self.test_name, 185 str(script)).load_module() 186 module.build(self) 187 else: 188 self.default_build() 189 190 def default_build( 191 self, 192 use_desugar=True, 193 use_hiddenapi=True, 194 need_dex=None, 195 zip_compression_method="deflate", 196 zip_align_bytes=None, 197 api_level:Union[int, str]=26, # Can also be named alias (string). 198 javac_args=[], 199 javac_classpath: List[Path]=[], 200 d8_flags=[], 201 smali_args=[], 202 use_smali=True, 203 use_jasmin=True, 204 ): 205 javac_classpath = javac_classpath.copy() # Do not modify default value. 206 207 # Wrap "pathlib.Path" with our own version that ensures all paths are absolute. 208 # Plain filenames are assumed to be relative to self.test_dir and made absolute. 209 class Path(pathlib.Path): 210 def __new__(cls, filename: str): 211 path = pathlib.Path(filename) 212 return path if path.is_absolute() else (self.test_dir / path) 213 214 need_dex = (self.host or self.target) if need_dex is None else need_dex 215 216 if self.jvm: 217 # No desugaring on jvm because it supports the latest functionality. 218 use_desugar = False 219 220 # Set API level for smali and d8. 221 if isinstance(api_level, str): 222 API_LEVEL = { 223 "default-methods": 24, 224 "parameter-annotations": 25, 225 "agents": 26, 226 "method-handles": 26, 227 "var-handles": 28, 228 } 229 api_level = API_LEVEL[api_level] 230 assert isinstance(api_level, int), api_level 231 232 def zip(zip_target: Path, *files: Path): 233 zip_args = ["-o", zip_target, "-C", zip_target.parent] 234 if zip_compression_method == "store": 235 zip_args.extend(["-L", "0"]) 236 for f in files: 237 zip_args.extend(["-f", f]) 238 self.soong_zip(zip_args) 239 240 if zip_align_bytes: 241 # zipalign does not operate in-place, so write results to a temp file. 242 with TemporaryDirectory() as tmp_dir: 243 tmp_file = Path(tmp_dir) / "aligned.zip" 244 self.zipalign(["-f", str(zip_align_bytes), zip_target, tmp_file]) 245 # replace original zip target with our temp file. 246 tmp_file.rename(zip_target) 247 248 249 def make_jasmin(dst_dir: Path, src_dir: Path) -> Optional[Path]: 250 if not use_jasmin or not src_dir.exists(): 251 return None # No sources to compile. 252 dst_dir.mkdir() 253 self.jasmin(["-d", dst_dir] + sorted(src_dir.glob("**/*.j"))) 254 return dst_dir 255 256 def make_smali(dst_dex: Path, src_dir: Path) -> Optional[Path]: 257 if not use_smali or not src_dir.exists(): 258 return None # No sources to compile. 259 self.smali(["-JXmx512m", "assemble"] + smali_args + ["--api", str(api_level)] + 260 ["--output", dst_dex] + sorted(src_dir.glob("**/*.smali"))) 261 return dst_dex 262 263 def make_java(dst_dir: Path, *src_dirs: Path) -> Optional[Path]: 264 if not any(src_dir.exists() for src_dir in src_dirs): 265 return None # No sources to compile. 266 dst_dir.mkdir(exist_ok=True) 267 args = self.javac_args.split(" ") + javac_args 268 args += ["-implicit:none", "-encoding", "utf8", "-d", dst_dir] 269 if not self.jvm: 270 args += ["-bootclasspath", self.bootclasspath] 271 if javac_classpath: 272 args += ["-classpath", javac_classpath] 273 for src_dir in src_dirs: 274 args += sorted(src_dir.glob("**/*.java")) 275 self.javac(args) 276 javac_post = Path("javac_post.sh") 277 if javac_post.exists(): 278 self.run(javac_post, [dst_dir]) 279 return dst_dir 280 281 282 # Make a "dex" file given a directory of classes. This will be 283 # packaged in a jar file. 284 def make_dex(src_dir: Path): 285 dst_jar = Path(src_dir.name + ".jar") 286 args = d8_flags + ["--min-api", str(api_level), "--output", dst_jar] 287 args += ["--lib", self.bootclasspath] if use_desugar else ["--no-desugaring"] 288 args += sorted(src_dir.glob("**/*.class")) 289 self.d8(args) 290 291 # D8 outputs to JAR files today rather than DEX files as DX used 292 # to. To compensate, we extract the DEX from d8's output to meet the 293 # expectations of make_dex callers. 294 dst_dex = Path(src_dir.name + ".dex") 295 with TemporaryDirectory() as tmp_dir: 296 zipfile.ZipFile(dst_jar, "r").extractall(tmp_dir) 297 (Path(tmp_dir) / "classes.dex").rename(dst_dex) 298 299 # Merge all the dex files. 300 # Skip non-existing files, but at least 1 file must exist. 301 def make_dexmerge(dst_dex: Path, *src_dexs: Path): 302 # Include destination. Skip any non-existing files. 303 srcs = [f for f in [dst_dex] + list(src_dexs) if f.exists()] 304 305 # NB: We merge even if there is just single input. 306 # It is useful to normalize non-deterministic smali output. 307 tmp_dir = self.test_dir / "dexmerge" 308 tmp_dir.mkdir() 309 self.d8(["--min-api", str(api_level), "--output", tmp_dir] + srcs) 310 assert not (tmp_dir / "classes2.dex").exists() 311 for src_file in srcs: 312 src_file.unlink() 313 (tmp_dir / "classes.dex").rename(dst_dex) 314 tmp_dir.rmdir() 315 316 317 def make_hiddenapi(*dex_files: Path): 318 if not use_hiddenapi or not Path("hiddenapi-flags.csv").exists(): 319 return # Nothing to do. 320 args: List[Union[str, Path]] = ["encode"] 321 for dex_file in dex_files: 322 args.extend(["--input-dex=" + str(dex_file), "--output-dex=" + str(dex_file)]) 323 args.append("--api-flags=hiddenapi-flags.csv") 324 args.append("--no-force-assign-all") 325 self.hiddenapi(args) 326 327 328 if Path("classes.dex").exists(): 329 zip(Path(self.test_name + ".jar"), Path("classes.dex")) 330 return 331 332 if Path("classes.dm").exists(): 333 zip(Path(self.test_name + ".jar"), Path("classes.dm")) 334 return 335 336 if make_jasmin(Path("jasmin_classes"), Path("jasmin")): 337 javac_classpath.append(Path("jasmin_classes")) 338 339 if make_jasmin(Path("jasmin_classes2"), Path("jasmin-multidex")): 340 javac_classpath.append(Path("jasmin_classes2")) 341 342 # To allow circular references, compile src/, src-multidex/, src-aotex/, 343 # src-bcpex/, src-ex/ together and pass the output as class path argument. 344 # Replacement sources in src-art/, src2/ and src-ex2/ can replace symbols 345 # used by the other src-* sources we compile here but everything needed to 346 # compile the other src-* sources should be present in src/ (and jasmin*/). 347 extra_srcs = ["src-multidex", "src-aotex", "src-bcpex", "src-ex"] 348 replacement_srcs = ["src2", "src-ex2"] + ([] if self.jvm else ["src-art"]) 349 if (Path("src").exists() and 350 any(Path(p).exists() for p in extra_srcs + replacement_srcs)): 351 make_java(Path("classes-tmp-all"), Path("src"), *map(Path, extra_srcs)) 352 javac_classpath.append(Path("classes-tmp-all")) 353 354 if make_java(Path("classes-aotex"), Path("src-aotex")) and need_dex: 355 make_dex(Path("classes-aotex")) 356 # rename it so it shows up as "classes.dex" in the zip file. 357 Path("classes-aotex.dex").rename(Path("classes.dex")) 358 zip(Path(self.test_name + "-aotex.jar"), Path("classes.dex")) 359 360 if make_java(Path("classes-bcpex"), Path("src-bcpex")) and need_dex: 361 make_dex(Path("classes-bcpex")) 362 # rename it so it shows up as "classes.dex" in the zip file. 363 Path("classes-bcpex.dex").rename(Path("classes.dex")) 364 zip(Path(self.test_name + "-bcpex.jar"), Path("classes.dex")) 365 366 make_java(Path("classes"), Path("src")) 367 368 if not self.jvm: 369 # Do not attempt to build src-art directories on jvm, 370 # since it would fail without libcore. 371 make_java(Path("classes"), Path("src-art")) 372 373 if make_java(Path("classes2"), Path("src-multidex")) and need_dex: 374 make_dex(Path("classes2")) 375 376 make_java(Path("classes"), Path("src2")) 377 378 # If the classes directory is not-empty, package classes in a DEX file. 379 # NB: some tests provide classes rather than java files. 380 if any(Path("classes").glob("*")) and need_dex: 381 make_dex(Path("classes")) 382 383 if Path("jasmin_classes").exists(): 384 # Compile Jasmin classes as if they were part of the classes.dex file. 385 if need_dex: 386 make_dex(Path("jasmin_classes")) 387 make_dexmerge(Path("classes.dex"), Path("jasmin_classes.dex")) 388 else: 389 # Move jasmin classes into classes directory so that they are picked up 390 # with -cp classes. 391 Path("classes").mkdir(exist_ok=True) 392 copytree(Path("jasmin_classes"), Path("classes"), dirs_exist_ok=True) 393 394 if need_dex and make_smali(Path("smali_classes.dex"), Path("smali")): 395 # Merge smali files into classes.dex, 396 # this takes priority over any jasmin files. 397 make_dexmerge(Path("classes.dex"), Path("smali_classes.dex")) 398 399 # Compile Jasmin classes in jasmin-multidex as if they were part of 400 # the classes2.jar 401 if Path("jasmin-multidex").exists(): 402 if need_dex: 403 make_dex(Path("jasmin_classes2")) 404 make_dexmerge(Path("classes2.dex"), Path("jasmin_classes2.dex")) 405 else: 406 # Move jasmin classes into classes2 directory so that 407 # they are picked up with -cp classes2. 408 Path("classes2").mkdir() 409 copytree(Path("jasmin_classes2"), Path("classes2"), dirs_exist_ok=True) 410 rmtree(Path("jasmin_classes2")) 411 412 if need_dex and make_smali(Path("smali_classes2.dex"), Path("smali-multidex")): 413 # Merge smali_classes2.dex into classes2.dex 414 make_dexmerge(Path("classes2.dex"), Path("smali_classes2.dex")) 415 416 make_java(Path("classes-ex"), Path("src-ex")) 417 418 make_java(Path("classes-ex"), Path("src-ex2")) 419 420 if Path("classes-ex").exists() and need_dex: 421 make_dex(Path("classes-ex")) 422 423 if need_dex and make_smali(Path("smali_classes-ex.dex"), Path("smali-ex")): 424 # Merge smali files into classes-ex.dex. 425 make_dexmerge(Path("classes-ex.dex"), Path("smali_classes-ex.dex")) 426 427 if Path("classes-ex.dex").exists(): 428 # Apply hiddenapi on the dex files if the test has API list file(s). 429 make_hiddenapi(Path("classes-ex.dex")) 430 431 # quick shuffle so that the stored name is "classes.dex" 432 Path("classes.dex").rename(Path("classes-1.dex")) 433 Path("classes-ex.dex").rename(Path("classes.dex")) 434 zip(Path(self.test_name + "-ex.jar"), Path("classes.dex")) 435 Path("classes.dex").rename(Path("classes-ex.dex")) 436 Path("classes-1.dex").rename(Path("classes.dex")) 437 438 # Apply hiddenapi on the dex files if the test has API list file(s). 439 if need_dex: 440 if any(Path(".").glob("*-multidex")): 441 make_hiddenapi(Path("classes.dex"), Path("classes2.dex")) 442 else: 443 make_hiddenapi(Path("classes.dex")) 444 445 # Create a single dex jar with two dex files for multidex. 446 if need_dex: 447 if Path("classes2.dex").exists(): 448 zip(Path(self.test_name + ".jar"), Path("classes.dex"), Path("classes2.dex")) 449 else: 450 zip(Path(self.test_name + ".jar"), Path("classes.dex")) 451 452 453# If we build just individual shard, we want to split the work among all the cores, 454# but if the build system builds all shards, we don't want to overload the machine. 455# We don't know which situation we are in, so as simple work-around, we use a lock 456# file to allow only one shard to use multiprocessing at the same time. 457def use_multiprocessing(mode: str) -> bool: 458 if "RBE_server_address" in os.environ: 459 return True 460 global lock_file 461 lock_path = Path(environ["TMPDIR"]) / ("art-test-run-test-build-py-" + mode) 462 lock_file = open(lock_path, "w") 463 try: 464 lockf(lock_file, LOCK_EX | LOCK_NB) 465 return True # We are the only instance of this script in the build system. 466 except BlockingIOError: 467 return False # Some other instance is already running. 468 469 470def main() -> None: 471 parser = ArgumentParser(description=__doc__) 472 parser.add_argument("--out", type=Path, help="Final zip file") 473 parser.add_argument("--mode", choices=["host", "jvm", "target"]) 474 parser.add_argument("--bootclasspath", type=Path) 475 parser.add_argument("--d8", type=Path) 476 parser.add_argument("--hiddenapi", type=Path) 477 parser.add_argument("--jasmin", type=Path) 478 parser.add_argument("--smali", type=Path) 479 parser.add_argument("--soong_zip", type=Path) 480 parser.add_argument("--zipalign", type=Path) 481 parser.add_argument("srcs", nargs="+", type=Path) 482 args = parser.parse_args() 483 484 android_build_top = Path(getcwd()).absolute() 485 ziproot = args.out.absolute().parent / "zip" 486 srcdirs = set(s.parents[-4].absolute() for s in args.srcs) 487 488 # Special hidden-api shard: If the --hiddenapi flag is provided, build only 489 # hiddenapi tests. Otherwise exclude all hiddenapi tests from normal shards. 490 def filter_by_hiddenapi(srcdir: Path) -> bool: 491 return (args.hiddenapi != None) == ("hiddenapi" in srcdir.name) 492 493 # Initialize the test objects. 494 # We need to do this before we change the working directory below. 495 tests: List[BuildTestContext] = [] 496 for srcdir in filter(filter_by_hiddenapi, srcdirs): 497 dstdir = ziproot / args.mode / srcdir.name 498 copytree(srcdir, dstdir) 499 tests.append(BuildTestContext(args, android_build_top, dstdir)) 500 501 # We can not change the working directory per each thread since they all run in parallel. 502 # Create invalid read-only directory to catch accidental use of current working directory. 503 with TemporaryDirectory("-do-not-use-cwd") as invalid_tmpdir: 504 os.chdir(invalid_tmpdir) 505 os.chmod(invalid_tmpdir, 0) 506 with ThreadPoolExecutor(cpu_count() if use_multiprocessing(args.mode) else 1) as pool: 507 jobs = {} 508 for ctx in tests: 509 jobs[ctx.test_name] = pool.submit(ctx.build) 510 for test_name, job in jobs.items(): 511 try: 512 job.result() 513 except Exception as e: 514 raise Exception("Failed to build " + test_name) from e 515 516 # Create the final zip file which contains the content of the temporary directory. 517 proc = run([android_build_top / args.soong_zip, "-o", android_build_top / args.out, 518 "-C", ziproot, "-D", ziproot], check=True) 519 520 521if __name__ == "__main__": 522 main() 523