1#!/usr/bin/python3 2# 3# Copyright (C) 2022 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import json 18import os 19import shutil 20import subprocess 21import sys 22from typing import List, Tuple 23 24import common 25from find_api_packages import ApiPackageFinder 26 27# Additional env vars to pass to the SOONG UI invocation. 28_ENV_VARS = { 29 # Orchestrator runs in a Read-only workspace, but inner_build does not 30 # (by default). Set this environment variable so that inner_build can 31 # setup its own nsjail. 32 "BUILD_BROKEN_SRC_DIR_IS_WRITABLE": "false", 33 34 # TODO: Once we are publishing a lightweight API surfaces tree, we 35 # should not need to set the environment variables. 36 "ALLOW_MISSING_DEPENDENCIES": "true", 37 "SKIP_VNDK_VARIANTS_CHECK": "true", 38} 39 40# Default TARGET_PRODUCT. See bazel/rules/make_injection.bzl, and envsetup.sh. 41DEFAULT_TARGET_PRODUCT = "aosp_arm" 42 43 44class InnerBuildSoong(common.Commands): 45 def __init__(self, env_vars=None): 46 """Initialize the instance. 47 48 Args: 49 env_vars: Environment variable updates. See common.setenv() 50 """ 51 self.env_vars = dict(**_ENV_VARS) 52 self.env_vars.update(env_vars or {}) 53 54 def export_api_contributions(self, args): 55 with common.setenv(**self.env_vars): 56 self._export_api_contributions(args) 57 58 def _export_api_contributions(self, args): 59 # Bazel is used to export API contributions even when the primary build 60 # system is Soong. 61 exporter = ApiExporterBazel(inner_tree=args.inner_tree, 62 out_dir=args.out_dir, 63 api_domains=args.api_domain) 64 exporter.export_api_contributions() 65 66 def analyze(self, args): 67 with common.setenv(**self.env_vars): 68 self._analyze(args) 69 70 def _analyze(self, args): 71 """Run analysis on this tree.""" 72 cmd = [ 73 "build/soong/soong_ui.bash", "--build-mode", 74 f"--dir={args.inner_tree}", "-all-modules", "nothing", 75 "--skip-soong-tests", "--search-api-dir", "--multitree-build" 76 ] 77 78 p = subprocess.run(cmd, shell=False, check=False) 79 if p.returncode: 80 sys.stderr.write(f"analyze: {cmd} failed with error message:\n" 81 f"{p.stderr.decode() if p.stderr else ''}") 82 sys.exit(p.returncode) 83 84 # Capture the environment variables passed by soong_ui to single-tree 85 # ninja. 86 env_path = os.path.join(args.out_dir, 'soong', 'ninja.environment') 87 with open(env_path, "r", encoding='iso-8859-1') as f: 88 try: 89 env_json = json.load(f) 90 except json.decoder.JSONDecodeError as ex: 91 sys.stderr.write(f"failed to parse {env_path}: {ex.msg}\n") 92 raise ex 93 shutil.copyfile(env_path, os.path.join(args.out_dir, "inner_tree.env")) 94 95 # Deliver the innertree's ninja file at `inner_tree.ninja`. 96 product = os.environ.get("TARGET_PRODUCT", DEFAULT_TARGET_PRODUCT) 97 src_path = os.path.join(args.out_dir, f"combined-{product}.ninja") 98 dst_path = os.path.join(args.out_dir, f"inner_tree.ninja") 99 shutil.copyfile(src_path, dst_path) 100 101 # TODO: Create an empty file for now. orchestrator will subninja the 102 # primary ninja file only if build_targets.json is not empty. 103 with open(os.path.join(args.out_dir, "build_targets.json"), 104 "w", 105 encoding='iso-8859-1') as f: 106 json.dump({"staging": []}, f, indent=2) 107 108 109class ApiMetadataFile(object): 110 """Utility class that wraps the generated API surface metadata files""" 111 112 def __init__(self, inner_tree: str, path: str, 113 bazel_output_user_root: str): 114 self.inner_tree = inner_tree 115 self.path = path 116 self.bazel_output_user_root = bazel_output_user_root 117 118 def fullpath(self) -> str: 119 # The Bazel convenience symlinks do not exist inside the nsjail 120 # workspace, since the workspace is read-only. 121 # Inject the output_user_root prefix into cquery result so that Build 122 # orchestrator can find the metadata files. 123 # 124 # e.g. cquery returns bazel-out/android_target-fastbuild/bin/... which 125 # does not exist. 126 # replace with <output_user_root>/... which does exist. 127 cleaned_path = self.path.replace("bazel-out", 128 self.bazel_output_user_root) 129 return os.path.join(self.inner_tree, cleaned_path) 130 131 def name(self) -> str: 132 """Returns filename""" 133 return os.path.basename(self.fullpath()) 134 135 def newerthan(self, otherpath: str) -> bool: 136 """Returns true if this file is newer than the file at `otherpath`""" 137 return not os.path.exists(otherpath) or os.path.getmtime( 138 otherpath) < os.path.getmtime(self.fullpath()) 139 140 141# ApiPackageFinder filters for special chars in .mcombo files 142_MCOMBO_WILDCARD_FILTERS = { 143 "*": lambda x: x.is_apex, 144 # TODO: Support more wildcards if necessary (e.g. vendor apex, google 145 # variants etc.) 146} 147 148 149class ApiExporterBazel(object): 150 """Generate API surface metadata files into a well-known directory 151 152 Intended Use: 153 This directory is subsequently scanned by the build orchestrator for API 154 surface assembly. 155 """ 156 157 def __init__(self, inner_tree: str, out_dir: str, api_domains: List[str]): 158 """Initialize the instance. 159 160 Args: 161 inner_tree: Root of the exporting tree 162 out_dir: output directory. The files will be copied to 163 $our_dir/api_contribtutions 164 api_domains: The API domains whose contributions should be exported 165 """ 166 self.inner_tree = inner_tree 167 self.out_dir = out_dir 168 self.api_domains = api_domains 169 170 def export_api_contributions(self): 171 contribution_targets = self._find_api_domain_contribution_targets() 172 metadata_files = self._build_api_domain_contribution_targets( 173 contribution_targets) 174 self._copy_api_domain_contribution_metadata_files(files=metadata_files) 175 176 def _find_api_domain_contribution_targets(self) -> List[str]: 177 """Return the label of the Bazel contribution targets to build""" 178 print(f"Finding api_domain_contribution Bazel BUILD targets " 179 f"in tree rooted at {self.inner_tree}") 180 finder = ApiPackageFinder(inner_tree_root=self.inner_tree) 181 contribution_targets = [] 182 for api_domain in self.api_domains: 183 default_name_filter = lambda x: x.api_domain == api_domain 184 api_domain_filter = _MCOMBO_WILDCARD_FILTERS.get( 185 api_domain, default_name_filter) 186 labels = finder.find_api_label_string_using_filter( 187 api_domain_filter) 188 contribution_targets.extend(labels) 189 return contribution_targets 190 191 def _build_api_domain_contribution_targets(self, 192 contribution_targets: List[str] 193 ) -> List[ApiMetadataFile]: 194 """Build the contribution targets 195 196 Return: 197 the filepath of the generated files. 198 """ 199 print(f"Running Bazel build on api_domain_contribution targets in " 200 f"tree rooted at {self.inner_tree}") 201 if not contribution_targets: 202 return None 203 self._run_bazel_cmd( 204 subcmd="build", 205 targets=contribution_targets, 206 capture_output=False, # log everything to terminal 207 ) 208 # Determine the output_user_root where the artifacts are created. 209 print("Running Bazel info in tree rooted at {self.inner_tree}") 210 proc = self._run_bazel_cmd( 211 subcmd="info", 212 targets=["output_path"], 213 capture_output=True, 214 run_bp2build=False, 215 ) 216 output_path = proc.stdout.decode().rstrip() 217 218 print("Running Bazel cquery on api_domain_contribution targets " 219 f"in tree rooted at {self.inner_tree}") 220 proc = self._run_bazel_cmd( 221 subcmd="cquery", 222 # cquery raises an error if multiple targets are provided. 223 # Create a union expression instead. 224 targets=[" union ".join(contribution_targets)], 225 subcmd_options=[ 226 "--output=files", 227 ], 228 capture_output=True, # parse cquery result from stdout 229 # we just ran bp2build. We can run it in again, 230 # but this adds time. 231 run_bp2build=False, 232 ) 233 # The cquery response contains a blank line at the end. 234 # Remove this before creating the filepaths array. 235 filepaths = proc.stdout.decode().rstrip().split("\n") 236 return [ 237 ApiMetadataFile(inner_tree=self.inner_tree, 238 path=filepath, 239 bazel_output_user_root=output_path) 240 for filepath in filepaths 241 ] 242 243 def _copy_api_domain_contribution_metadata_files( 244 self, files: List[ApiMetadataFile]): 245 """Copies the metadata files to a well-known location""" 246 target_dir = os.path.join(self.out_dir, "api_contributions") 247 print(f"Copying API contribution metadata files of tree rooted at " 248 f"{self.inner_tree} to {target_dir}") 249 # Create the directory if it does not exist, even if that inner_tree has 250 # no contributions. 251 if not os.path.exists(target_dir): 252 os.makedirs(target_dir) 253 if not files: 254 return 255 # Delete stale API contribution files 256 filenames = {file.name() for file in files} 257 with os.scandir(target_dir) as it: 258 for dirent in it: 259 if dirent.name not in filenames: 260 os.remove(dirent.path) 261 # Copy API contribution files if mtime has changed 262 for file in files: 263 target = os.path.join(target_dir, file.name()) 264 if file.newerthan(target): 265 # Copy file without metadata like read-only 266 shutil.copyfile(file.fullpath(), target) 267 268 def _run_bazel_cmd(self, 269 subcmd: str, 270 targets: List[str], 271 subcmd_options: Tuple[str] = (), 272 run_bp2build=True, 273 **kwargs) -> subprocess.CompletedProcess: 274 """Runs Bazel subcmd with Multi-tree specific configuration""" 275 # TODO (b/244766775): Replace the two discrete cmds once the new 276 # b-equivalent entrypoint is available. 277 if run_bp2build: 278 self._run_bp2build_cmd() 279 output_user_root = self._output_user_root() 280 cmd = [ 281 # Android's Bazel-entrypoint. Contains configs like the JDK to use. 282 "build/bazel/bin/bazel", 283 subcmd, 284 # Run Bazel on the synthetic api_bp2build workspace. 285 "--config=api_bp2build", 286 "--config=android", 287 f"--symlink_prefix={output_user_root}", # Use prefix hack to create the convenience symlinks in out/ 288 ] 289 subcmd_options = list(subcmd_options) 290 cmd += subcmd_options + targets 291 return self._run_cmd(cmd, **kwargs) 292 293 # Create a unique output root for this workspace inside the nsjail. 294 # This ensures that we do not share a single Bazel server between the 295 # workspace inside and outside the nsjail. 296 def _output_user_root(self) -> str: 297 return os.path.join(self.inner_tree, self.out_dir, "bazel") 298 299 def _run_bp2build_cmd(self, **kwargs) -> subprocess.CompletedProcess: 300 """Runs b2pbuild to generate the synthetic Bazel workspace""" 301 cmd = [ 302 "build/soong/soong_ui.bash", 303 "--build-mode", 304 "--all-modules", 305 f"--dir={self.inner_tree}", 306 "api_bp2build", 307 "--skip-soong-tests", 308 "--multitree-build", 309 "--search-api-dir", # This ensures that Android.bp.list remains the same in the analysis step. 310 ] 311 return self._run_cmd(cmd, **kwargs) 312 313 def _run_cmd(self, cmd, **kwargs) -> subprocess.CompletedProcess: 314 proc = subprocess.run(cmd, 315 cwd=self.inner_tree, 316 shell=False, 317 check=False, 318 **kwargs) 319 if proc.returncode: 320 sys.stderr.write( 321 f"export_api_contributions: {cmd} failed with error message:\n" 322 ) 323 if proc.stderr: 324 sys.stderr.write(proc.stderr.decode()) 325 sys.exit(proc.returncode) 326 return proc 327 328 329def main(argv): 330 return InnerBuildSoong().Run(argv) 331 332 333if __name__ == "__main__": 334 sys.exit(main(sys.argv)) 335