1#!/usr/bin/python3 2# 3# Copyright (C) 2022 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import enum 18import json 19import os 20import subprocess 21import sys 22import textwrap 23 24import nsjail 25 26_INNER_BUILD = ".inner_build" 27 28 29class InnerTreeKey(object): 30 """Trees are identified uniquely by their root and the TARGET_PRODUCT they will use to build. 31 If a single tree uses two different prdoucts, then we won't make assumptions about 32 them sharing _anything_. 33 TODO: This is true for soong. It's more likely that bazel could do analysis for two 34 products at the same time in a single tree, so there's an optimization there to do 35 eventually.""" 36 37 def __init__(self, root, product): 38 if isinstance(root, list): 39 self.melds = root[1:] 40 root = root[0] 41 else: 42 self.melds = [] 43 self.root = root 44 self.product = product 45 46 def __str__(self): 47 return (f"TreeKey(root={enquote(self.root)} " 48 f"product={enquote(self.product)}") 49 50 def __hash__(self): 51 return hash((self.root, str(self.melds), self.product)) 52 53 def _cmp(self, other): 54 assert isinstance(other, InnerTreeKey) 55 if self.root < other.root: 56 return -1 57 if self.root > other.root: 58 return 1 59 if self.melds < other.melds: 60 return -1 61 if self.melds > other.melds: 62 return 1 63 if self.product == other.product: 64 return 0 65 if self.product is None: 66 return -1 67 if other.product is None: 68 return 1 69 if self.product < other.product: 70 return -1 71 return 1 72 73 def __eq__(self, other): 74 return self._cmp(other) == 0 75 76 def __ne__(self, other): 77 return self._cmp(other) != 0 78 79 def __lt__(self, other): 80 return self._cmp(other) < 0 81 82 def __le__(self, other): 83 return self._cmp(other) <= 0 84 85 def __gt__(self, other): 86 return self._cmp(other) > 0 87 88 def __ge__(self, other): 89 return self._cmp(other) >= 0 90 91 92class InnerTree(object): 93 def __init__(self, context, paths, product, variant): 94 """Initialize with the inner tree root (relative to the workspace root)""" 95 if not isinstance(paths, list): 96 paths = [paths] 97 self.root = paths[0] 98 self.meld_dirs = paths[1:] 99 # TODO: more complete checking (include '../' in the checks, etc. 100 if any(x.startswith(os.path.sep) for x in self.meld_dirs): 101 raise Exception( 102 f"meld directories may not start with {os.path.sep}") 103 if any(x.startswith('=') for x in self.meld_dirs[1:]): 104 raise Exception('only the first meld directory can specify "="') 105 106 self.product = product 107 self.variant = variant 108 self.domains = {} 109 self.context = context 110 self.env_used = [] 111 self.nsjail = context.tools.nsjail 112 self.out_root_origin = context.out.inner_tree_dir(self.root, product) 113 self.out = OutDirLayout(self.root, self.out_root_origin) 114 self._meld_config = None 115 116 def __str__(self): 117 return (f"InnerTree(root={enquote(self.root)} " 118 f"product={enquote(self.product)} " 119 f"domains={enquote(list(self.domains.keys()))} " 120 f"meld={enquote(self.meld_dirs)})") 121 122 @property 123 def meld_config(self): 124 """Return the meld configuration for invoking inner_build.""" 125 if self._meld_config: 126 return self._meld_config 127 128 inner_tree_src_path = os.path.abspath(self.root) 129 config = nsjail.Nsjail(inner_tree_src_path) 130 inner_tree_out_path = self.out.root(base=self.out.Base.OUTER, 131 abspath=True) 132 out_root_origin = self.out.root() 133 134 # Add TARGET_PRODUCT and TARGET_BUILD_VARIANT. 135 if self.product: 136 config.add_envar(name="TARGET_PRODUCT", value=self.product) 137 config.add_envar(name="TARGET_BUILD_VARIANT", value=self.variant) 138 139 # TODO: determine what other envirnoment variables need to be copied 140 # into the nsjail config. 141 142 # If the first meld dir path starts with "=", overlay the entire tree 143 # with that before melding other sub manifests. 144 meld_dirs = self.meld_dirs 145 tree_root = inner_tree_src_path 146 if meld_dirs and meld_dirs[0].startswith('='): 147 tree_root = os.path.abspath(meld_dirs[0][1:]) 148 meld_dirs = meld_dirs[1:] 149 sys.stderr.write(f'overlaying {self.root} with {tree_root}\n') 150 151 config.add_mountpt(src=tree_root, 152 dst=inner_tree_src_path, 153 is_bind=True, 154 rw=False, 155 mandatory=True) 156 # Place OUTDIR at /out 157 os.makedirs(out_root_origin, exist_ok=True) 158 config.add_mountpt(src=os.path.abspath(out_root_origin), 159 dst=inner_tree_out_path, 160 is_bind=True, 161 rw=True, 162 mandatory=True) 163 164 # TODO: Once we have the lightweight tree, this mount should move to 165 # platform/apisurfaces, and be mandatory. 166 api_surfaces = self.context.out.api_surfaces_dir( 167 base=self.context.out.Base.ORIGIN, abspath=True) 168 # Always mount api_surfaces dir. 169 # The mount point is out/api_surfaces -> <inner_tree>/out/api_surfaces 170 # soong_finder will be speciall-cased to look for Android.bp files in 171 # this dir. 172 api_surfaces_inner_tree = os.path.join(inner_tree_out_path, 173 "api_surfaces") 174 os.makedirs(api_surfaces, exist_ok=True) 175 os.makedirs(api_surfaces_inner_tree, exist_ok=True) 176 config.add_mountpt(src=api_surfaces, 177 dst=api_surfaces_inner_tree, 178 is_bind=True, 179 rw=False, 180 mandatory=False) 181 # Share the Network namespace for API export. 182 # This ensures that the Bazel client can communicate with the Bazel daemon. 183 # This does not preclude build systems of inner trees from setting 184 # up different sandbox configs. e.g. Soong is free to run the build 185 # in a sandbox that disables network access. 186 # TODO: Make this more restrictive. This should only be limited to the 187 # loopback device. 188 config.add_option(name="clone_newnet", value="false") 189 190 def _meld_git(shared, src): 191 dst = os.path.join(self.root, src[len(shared) + 1:]) 192 abs_dst = os.path.join(inner_tree_src_path, src[len(shared) + 1:]) 193 abs_src = os.path.abspath(src) 194 # Only meld if we have not already mounted something at {dst}, and 195 # either the project is missing, or is an empty directory: nsjail 196 # creates empty directories when it mounts the directory. 197 if abs_dst in config.mount_points: 198 sys.stderr.write(f'{dst} already mounted, ignoring {src}\n') 199 elif not os.path.isdir(dst) or not os.listdir(dst): 200 # TODO: For repo workspaces, we need to handle <linkfile/> and 201 # <copyfile/> elements from the manifest. 202 sys.stderr.write(f'melding {src} into {dst}\n') 203 config.add_mountpt(src=abs_src, 204 dst=abs_dst, 205 is_bind=True, 206 rw=False, 207 mandatory=True) 208 209 for shared in meld_dirs: 210 if os.path.isdir(os.path.join(shared, '.git')): 211 # TODO: If this is the root of the meld_dir, process the 212 # modules instead of the git project. 213 print('TODO: handle git submodules case') 214 continue 215 216 # Use os.walk (which uses os.scandir), so that we get recursion 217 # for free. 218 for src, dirs, _ in os.walk(shared): 219 # When repo syncs the workspace, .git is a symlink. 220 if '.git' in dirs or os.path.isdir(os.path.join(src, '.git')): 221 _meld_git(shared, src) 222 # Stop recursing. 223 dirs[:] = [] 224 # TODO: determine what other source control systems we need 225 # to detect and support here. 226 227 self._meld_config = config 228 return self._meld_config 229 230 @property 231 def build_domains(self): 232 """The build_domains for this inner-tree.""" 233 return sorted(self.domains.keys()) 234 235 def set_env_used(self): 236 """Record the environment used in the inner tree.""" 237 with open(self.out.env_used_file(), "r", encoding="iso-8859-1") as f: 238 try: 239 self.env_used = json.load(f) 240 except json.decoder.JSONDecodeError as ex: 241 sys.stderr.write(f"failed to parse {env_path}: {ex.msg}\n") 242 raise ex 243 244 def invoke(self, args): 245 """Call the inner tree command for this inner tree. Exits on failure.""" 246 # TODO: Build time tracing 247 248 # Validate that there is a .inner_build command to run at the root of the tree 249 # so we can print a good error message 250 # If we are melding the inner_build into the tree, it won't be 251 # executable at this time. 252 #inner_build_tool = os.path.join(self.root, _INNER_BUILD) 253 #if not os.path.exists(inner_build_tool): 254 # sys.stderr.write( 255 # f"Unable to execute {inner_build_tool}. Is there an inner tree " 256 # f"or lunch combo misconfiguration?\n") 257 # sys.exit(1) 258 259 meld_config = self.meld_config 260 inner_tree_src_path = meld_config.cwd 261 262 # Write the nsjail config 263 nsjail_config_file = self.out.nsjail_config_file() 264 meld_config.generate_config(nsjail_config_file) 265 266 # Build the command 267 cmd = [ 268 self.nsjail, 269 "--config", 270 nsjail_config_file, 271 "--", 272 os.path.join(inner_tree_src_path, _INNER_BUILD), 273 "--out_dir", 274 self.out.root(base=self.out.Base.INNER), 275 ] 276 cmd += args 277 278 # Run the command 279 print("% " + " ".join(cmd)) 280 process = subprocess.run(cmd, shell=False, check=False) 281 282 # TODO: Probably want better handling of inner tree failures 283 if process.returncode: 284 sys.stderr.write( 285 f"Build error in inner tree: {self.root}\nstopping " 286 f"multitree build.\n") 287 sys.exit(1) 288 289 290class InnerTrees(object): 291 def __init__(self, trees, domains): 292 self.trees = trees 293 self.domains = domains 294 295 def __str__(self): 296 """Return a debugging dump of this object""" 297 298 def _vals(values): 299 return ("\n" + " " * 16).join(sorted([str(t) for t in values])) 300 301 return textwrap.dedent(f"""\ 302 InnerTrees {{ 303 inner-tree: [ 304 {self.trees.values()[0]} 305 {_vals(self.trees.values()[1:])} 306 ] 307 domains: [ 308 {_vals(self.domains.values())} 309 ] 310 }}""") 311 312 def __iter__(self): 313 """Return a generator yielding the sorted inner tree keys.""" 314 for key in sorted(self.trees.keys()): 315 yield key 316 317 def for_each_tree(self, func, cookie=None): 318 """Call func for each of the inner trees once for each product that will be built in it. 319 320 The calls will be in a stable order. 321 322 Return a map of the InnerTreeKey to the return value from func(). 323 """ 324 result = {x: func(x, self.trees[x], cookie) for x in self} 325 return result 326 327 def get(self, tree_key): 328 """Get an inner tree for tree_key""" 329 return self.trees.get(tree_key) 330 331 def keys(self): 332 """Get the keys for the inner trees in name order.""" 333 return [self.trees[k] for k in sorted(self.trees.keys())] 334 335 336@enum.unique 337class OutDirBase(enum.Enum): 338 """The basepath to use for output paths. 339 340 ORIGIN: Path is relative to ${OUT_DIR}. Use this when the path will be 341 consumed while not nsjailed. (default) 342 OUTER: Path is relative to the outer tree root. Use this when the path 343 will be consumed while nsjailed in the outer tree. 344 INNER: Path is relative to the inner tree root. Use this when the path 345 will be consumed while nsjailed in the inner tree. 346 """ 347 DEFAULT = 0 348 ORIGIN = 1 349 OUTER = 2 350 INNER = 3 351 352 353class OutDirLayout(object): 354 """Encapsulates the logic about the layout of the inner tree out directories. 355 See also context.OutDir for outer tree out dir contents.""" 356 357 # For ease of use. 358 Base = OutDirBase 359 360 def __init__(self, tree_root, out_origin, out_path="out"): 361 """Initialize with the root of the OUT_DIR for the inner tree. 362 363 Args: 364 tree_root: The workspace-relative path of the inner_tree. 365 out_origin: The OUT_DIR path for the inner tree. 366 Usually "${OUT_DIR}/trees/{tree_root}_{product}" 367 out_path: Where the inner tree out_dir will be mapped, relative to the 368 inner tree root. Usually "out". 369 """ 370 self._base = {} 371 self._base[self.Base.ORIGIN] = out_origin 372 self._base[self.Base.OUTER] = os.path.join(tree_root, out_path) 373 self._base[self.Base.INNER] = out_path 374 self._base[self.Base.DEFAULT] = self._base[self.Base.ORIGIN] 375 376 def _generate_path(self, 377 *args, 378 base: OutDirBase = OutDirBase.DEFAULT, 379 abspath=False): 380 """Return the path to the file. 381 382 Args: 383 relpath: The inner tree out_dir relative path to use. 384 base: Which base path to use. 385 abspath: return the absolute path. 386 """ 387 ret = os.path.join(self._base[base], *args) 388 if abspath: 389 ret = os.path.abspath(ret) 390 return ret 391 392 def root(self, *args, **kwargs): 393 return self._generate_path(*args, **kwargs) 394 395 def api_contributions_dir(self, **kwargs): 396 return self._generate_path("api_contributions", **kwargs) 397 398 def build_targets_file(self, **kwargs): 399 return self._generate_path("build_targets.json", **kwargs) 400 401 def env_used_file(self, **kwargs): 402 return self._generate_path("inner_tree.env", **kwargs) 403 404 def main_ninja_file(self, **kwargs): 405 return self._generate_path("inner_tree.ninja", **kwargs) 406 407 def nsjail_config_file(self, **kwargs): 408 return self._generate_path("nsjail.cfg", **kwargs) 409 410 def tree_info_file(self, **kwargs): 411 return self._generate_path("tree_info.json", **kwargs) 412 413 def tree_query(self, **kwargs): 414 return self._generate_path("tree_query.json", **kwargs) 415 416 417def enquote(s): 418 return json.dumps(s) 419