1# Copyright 2020 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Mounts all the projects required by a selected Build target. 16 17For details on how filesystem overlays work see the filesystem overlays 18section of the README.md. 19""" 20 21from __future__ import absolute_import 22from __future__ import division 23from __future__ import print_function 24 25import collections 26import os 27import subprocess 28import tempfile 29import xml.etree.ElementTree as ET 30from . import config 31 32BindMount = collections.namedtuple( 33 'BindMount', ['source_dir', 'readonly', 'allows_replacement']) 34 35 36class BindOverlay(object): 37 """Manages filesystem overlays of Android source tree using bind mounts. 38 """ 39 40 MAX_BIND_MOUNTS = 10000 41 42 def _HideDir(self, target_dir): 43 """Temporarily replace the target directory for an empty directory. 44 45 Args: 46 target_dir: A string path to the target directory. 47 48 Returns: 49 A string path to the empty directory that replaced the target directory. 50 """ 51 empty_dir = tempfile.mkdtemp(prefix='empty_dir_') 52 self._AddBindMount(empty_dir, target_dir) 53 return empty_dir 54 55 def _FindBindMountConflict(self, path): 56 """Finds any path in the bind mounts that conflicts with the provided path. 57 58 Args: 59 path: A string path to be checked. 60 61 Returns: 62 A tuple containing a string of the conflicting path in the bind mounts and 63 whether or not to allow this path to supersede any conflicts. 64 None, False if there was no conflict found. 65 """ 66 conflict_path = None 67 allows_replacement = False 68 for bind_destination, bind_mount in self._bind_mounts.items(): 69 allows_replacement = bind_mount.allows_replacement 70 # Check if the path is a subdir or the bind destination 71 if path == bind_destination: 72 conflict_path = bind_mount.source_dir 73 break 74 elif path.startswith(bind_destination + os.sep): 75 relative_path = os.path.relpath(path, bind_destination) 76 path_in_source = os.path.join(bind_mount.source_dir, relative_path) 77 if os.path.exists(path_in_source) and os.listdir(path_in_source): 78 # A conflicting path exists within this bind mount 79 # and it's not empty 80 conflict_path = path_in_source 81 break 82 83 return conflict_path, allows_replacement 84 85 def _AddOverlay(self, source_dir, overlay_dir, intermediate_work_dir, 86 skip_subdirs, allowed_projects, destination_dir, 87 allowed_read_write, contains_read_write, 88 is_replacement_allowed): 89 """Adds a single overlay directory. 90 91 Args: 92 source_dir: A string with the path to the Android platform source. 93 overlay_dir: A string path to the overlay directory to apply. 94 intermediate_work_dir: A string path to the intermediate work directory used as the 95 base for constructing the overlay filesystem. 96 skip_subdirs: A set of string paths to skip from overlaying. 97 allowed_projects: If not None, any .git project path not in this list 98 is excluded from overlaying. 99 destination_dir: A string with the path to the source with the overlays 100 applied to it. 101 allowed_read_write: A function returns true if the path input should 102 be allowed read/write access. 103 contains_read_write: A function returns true if the path input contains 104 a sub-path that should be allowed read/write access. 105 is_replacement_allowed: A function returns true if the path can replace a 106 subsequent path. 107 """ 108 # Traverse the overlay directory twice 109 # The first pass only process git projects 110 # The second time process all other files that are not in git projects 111 112 # We need to process all git projects first because 113 # the way we process a non-git directory will depend on if 114 # it contains a git project in a subdirectory or not. 115 116 dirs_with_git_projects = set('/') 117 for current_dir_origin, subdirs, files in os.walk(overlay_dir): 118 119 if current_dir_origin in skip_subdirs: 120 del subdirs[:] 121 continue 122 123 current_dir_relative = os.path.relpath(current_dir_origin, overlay_dir) 124 current_dir_destination = os.path.normpath( 125 os.path.join(destination_dir, current_dir_relative)) 126 127 if '.git' in subdirs or '.git' in files or '.bindmount' in files: 128 # The current dir is a git project 129 # so just bind mount it 130 del subdirs[:] 131 132 if '.bindmount' in files or (not allowed_projects or 133 os.path.relpath(current_dir_origin, source_dir) in allowed_projects): 134 self._AddBindMount( 135 current_dir_origin, current_dir_destination, 136 False if allowed_read_write(current_dir_origin) else True, 137 is_replacement_allowed( 138 os.path.basename(overlay_dir), current_dir_relative)) 139 140 current_dir_ancestor = current_dir_origin 141 while current_dir_ancestor and current_dir_ancestor not in dirs_with_git_projects: 142 dirs_with_git_projects.add(current_dir_ancestor) 143 current_dir_ancestor = os.path.dirname(current_dir_ancestor) 144 145 # Process all other files that are not in git projects 146 for current_dir_origin, subdirs, files in os.walk(overlay_dir): 147 148 if current_dir_origin in skip_subdirs: 149 del subdirs[:] 150 continue 151 152 if '.git' in subdirs or '.git' in files or '.bindmount' in files: 153 del subdirs[:] 154 continue 155 156 current_dir_relative = os.path.relpath(current_dir_origin, overlay_dir) 157 current_dir_destination = os.path.normpath( 158 os.path.join(destination_dir, current_dir_relative)) 159 160 bindCurrentDir = True 161 162 # Directories with git projects can't be bind mounted 163 # because git projects are individually mounted 164 if current_dir_origin in dirs_with_git_projects: 165 bindCurrentDir = False 166 167 # A directory that contains read-write paths should only 168 # ever be bind mounted if the directory itself is read-write 169 if contains_read_write(current_dir_origin) and not allowed_read_write(current_dir_origin): 170 bindCurrentDir = False 171 172 if bindCurrentDir: 173 # The current dir can be bind mounted wholesale 174 del subdirs[:] 175 if allowed_read_write(current_dir_origin): 176 self._AddBindMount(current_dir_origin, current_dir_destination, False) 177 else: 178 self._AddBindMount(current_dir_origin, current_dir_destination, True) 179 continue 180 181 # If we've made it this far then we're going to process 182 # each file and subdir individually 183 184 for subdir in subdirs: 185 subdir_origin = os.path.join(current_dir_origin, subdir) 186 # Symbolic links to subdirectories 187 # have to be copied to the intermediate work directory. 188 # We can't bind mount them because bind mounts dereference 189 # symbolic links, and the build system filters out any 190 # directory symbolic links. 191 if os.path.islink(subdir_origin): 192 if subdir_origin not in skip_subdirs: 193 subdir_destination = os.path.join(intermediate_work_dir, 194 current_dir_relative, subdir) 195 self._CopyFile(subdir_origin, subdir_destination) 196 197 # bind each file individually then keep traversing 198 for file in files: 199 file_origin = os.path.join(current_dir_origin, file) 200 file_destination = os.path.join(current_dir_destination, file) 201 if allowed_read_write(file_origin): 202 self._AddBindMount(file_origin, file_destination, False) 203 else: 204 self._AddBindMount(file_origin, file_destination, True) 205 206 207 def _AddArtifactDirectories(self, source_dir, destination_dir, skip_subdirs): 208 """Add directories that were not synced as workspace source. 209 210 Args: 211 source_dir: A string with the path to the Android platform source. 212 destination_dir: A string with the path to the source where the overlays 213 will be applied. 214 skip_subdirs: A set of string paths to be skipped from overlays. 215 216 Returns: 217 A list of string paths to be skipped from overlaying. 218 """ 219 220 # Ensure the main out directory exists 221 main_out_dir = os.path.join(source_dir, 'out') 222 if not os.path.exists(main_out_dir): 223 os.makedirs(main_out_dir) 224 225 for subdir in os.listdir(source_dir): 226 if subdir.startswith('out'): 227 out_origin = os.path.join(source_dir, subdir) 228 if out_origin in skip_subdirs: 229 continue 230 out_destination = os.path.join(destination_dir, subdir) 231 self._AddBindMount(out_origin, out_destination, False) 232 skip_subdirs.add(out_origin) 233 234 repo_origin = os.path.join(source_dir, '.repo') 235 if os.path.exists(repo_origin): 236 repo_destination = os.path.normpath( 237 os.path.join(destination_dir, '.repo')) 238 self._AddBindMount(repo_origin, repo_destination, True) 239 skip_subdirs.add(repo_origin) 240 241 return skip_subdirs 242 243 def _AddOverlays(self, source_dir, overlay_dirs, destination_dir, 244 skip_subdirs, allowed_projects, allowed_read_write, 245 contains_read_write, is_replacement_allowed): 246 """Add the selected overlay directories. 247 248 Args: 249 source_dir: A string with the path to the Android platform source. 250 overlay_dirs: A list of strings with the paths to the overlay 251 directory to apply. 252 destination_dir: A string with the path to the source where the overlays 253 will be applied. 254 skip_subdirs: A set of string paths to be skipped from overlays. 255 allowed_projects: If not None, any .git project path not in this list 256 is excluded from overlaying. 257 allowed_read_write: A function returns true if the path input should 258 be allowed read/write access. 259 contains_read_write: A function returns true if the path input contains 260 a sub-path that should be allowed read/write access. 261 is_replacement_allowed: A function returns true if the path can replace a 262 subsequent path. 263 """ 264 265 # Create empty intermediate workdir 266 intermediate_work_dir = self._HideDir(destination_dir) 267 overlay_dirs.append(source_dir) 268 269 skip_subdirs = self._AddArtifactDirectories(source_dir, destination_dir, 270 skip_subdirs) 271 272 273 # Bind mount each overlay directory using a 274 # depth first traversal algorithm. 275 # 276 # The algorithm described works under the condition that the overlaid file 277 # systems do not have conflicting projects or that the conflict path is 278 # specifically called-out as a replacement path. 279 # 280 # The results of attempting to overlay two git projects on top 281 # of each other are unpredictable and may push the limits of bind mounts. 282 283 skip_subdirs.add(os.path.join(source_dir, 'overlays')) 284 285 for overlay_dir in overlay_dirs: 286 self._AddOverlay(source_dir, overlay_dir, intermediate_work_dir, 287 skip_subdirs, allowed_projects, destination_dir, 288 allowed_read_write, contains_read_write, 289 is_replacement_allowed) 290 291 292 def _AddBindMount(self, 293 source_dir, 294 destination_dir, 295 readonly=False, 296 allows_replacement=False): 297 """Adds a bind mount for the specified directory. 298 299 Args: 300 source_dir: A string with the path of a source directory to bind. 301 It must already exist. 302 destination_dir: A string with the path ofa destination 303 directory to bind the source into. If it does not exist, 304 it will be created. 305 readonly: A flag to indicate whether this path should be bind mounted 306 with read-only access. 307 allow_replacement: A flag to indicate whether this path is allowed to replace a 308 conflicting path. 309 """ 310 conflict_path, replacement = self._FindBindMountConflict(destination_dir) 311 if conflict_path and not replacement: 312 raise ValueError("Project %s could not be overlaid at %s " 313 "because it conflicts with %s" 314 % (source_dir, destination_dir, conflict_path)) 315 elif not conflict_path: 316 if len(self._bind_mounts) >= self.MAX_BIND_MOUNTS: 317 raise ValueError("Bind mount limit of %s reached" % self.MAX_BIND_MOUNTS) 318 self._bind_mounts[destination_dir] = BindMount( 319 source_dir=source_dir, 320 readonly=readonly, 321 allows_replacement=allows_replacement) 322 323 def _CopyFile(self, source_path, dest_path): 324 """Copies a file to the specified destination. 325 326 Args: 327 source_path: A string with the path of a source file to copy. It must 328 exist. 329 dest_path: A string with the path to copy the file to. It should not 330 exist. 331 """ 332 dest_dir = os.path.dirname(dest_path) 333 if not os.path.exists(dest_dir): 334 os.makedirs(dest_dir) 335 subprocess.check_call(['cp', '--no-dereference', source_path, dest_path]) 336 337 def GetBindMounts(self): 338 """Enumerates all bind mounts required by this Overlay. 339 340 Returns: 341 An ordered dict of BindMount objects keyed by destination path string. 342 The order of the bind mounts does matter, this is why it's an ordered 343 dict instead of a standard dict. 344 """ 345 return self._bind_mounts 346 347 def _GetReadWriteFunction(self, build_config, source_dir): 348 """Returns a function that tells you how to mount a path. 349 350 Args: 351 build_config: A config.BuildConfig instance of the build target to be 352 prepared. 353 source_dir: A string with the path to the Android platform source. 354 355 Returns: 356 A function that takes a string path as an input and returns 357 True if the path should be mounted read-write or False if 358 the path should be mounted read-only. 359 """ 360 361 # The read/write allowlist provides paths relative to the source dir. It 362 # needs to be updated with absolute paths to make lookup possible. 363 rw_allowlist = {os.path.join(source_dir, p) for p in build_config.allow_readwrite} 364 365 def AllowReadWrite(path): 366 return build_config.allow_readwrite_all or path in rw_allowlist 367 368 return AllowReadWrite 369 370 def _GetContainsReadWriteFunction(self, build_config, source_dir): 371 """Returns a function that tells you if a directory contains a read-write dir 372 373 Args: 374 build_config: A config.BuildConfig instance of the build target to be 375 prepared. 376 source_dir: A string with the path to the Android platform source. 377 378 Returns: 379 A function that takes a string path as an input and returns 380 True if the path contains a read-write path 381 """ 382 383 # Get all dirs with allowed read-write 384 # and all their ancestor directories 385 contains_rw = set() 386 for path in build_config.allow_readwrite: 387 while path not in ["", "/"]: 388 # The read/write allowlist provides paths relative to the source dir. It 389 # needs to be updated with absolute paths to make lookup possible. 390 contains_rw.add(os.path.join(source_dir, path)) 391 path = os.path.dirname(path) 392 393 def ContainsReadWrite(path): 394 return build_config.allow_readwrite_all or path in contains_rw 395 396 return ContainsReadWrite 397 398 def _GetAllowedProjects(self, build_config): 399 """Returns a set of paths that are allowed to contain .git projects. 400 401 Args: 402 build_config: A config.BuildConfig instance of the build target to be 403 prepared. 404 405 Returns: 406 If the target has an allowed projects file: a set of paths. Any .git 407 project path not in this set should be excluded from overlaying. 408 Otherwise: None 409 """ 410 if not build_config.allowed_projects_file: 411 return None 412 allowed_projects = ET.parse(build_config.allowed_projects_file) 413 paths = set() 414 for child in allowed_projects.getroot().findall("project"): 415 paths.add(child.attrib.get("path", child.attrib["name"])) 416 return paths 417 418 def _IsReplacementAllowedFunction(self, build_config): 419 """Returns a function to determin if a given path is replaceable. 420 421 Args: 422 build_config: A config.BuildConfig instance of the build target to be 423 prepared. 424 425 Returns: 426 A function that takes an overlay name and string path as input and 427 returns True if the path is replaceable. 428 """ 429 def is_replacement_allowed_func(overlay_name, path): 430 for overlay in build_config.overlays: 431 if overlay_name == overlay.name and path in overlay.replacement_paths: 432 return True 433 return False 434 435 return is_replacement_allowed_func 436 437 def __init__(self, 438 build_target, 439 source_dir, 440 cfg, 441 whiteout_list = [], 442 destination_dir=None, 443 quiet=False): 444 """Inits Overlay with the details of what is going to be overlaid. 445 446 Args: 447 build_target: A string with the name of the build target to be prepared. 448 source_dir: A string with the path to the Android platform source. 449 cfg: A config.Config instance. 450 whiteout_list: A list of directories to hide from the build system. 451 destination_dir: A string with the path where the overlay filesystem 452 will be created. If none is provided, the overlay filesystem 453 will be applied directly on top of source_dir. 454 quiet: A boolean that, when True, suppresses debug output. 455 """ 456 self._quiet = quiet 457 458 if not destination_dir: 459 destination_dir = source_dir 460 461 self._overlay_dirs = None 462 # The order of the bind mounts does matter, this is why it's an ordered 463 # dict instead of a standard dict. 464 self._bind_mounts = collections.OrderedDict() 465 466 # We will be repeateadly searching for items to skip so a set 467 # seems appropriate 468 skip_subdirs = set(whiteout_list) 469 470 build_config = cfg.get_build_config(build_target) 471 472 allowed_read_write = self._GetReadWriteFunction(build_config, source_dir) 473 contains_read_write = self._GetContainsReadWriteFunction(build_config, source_dir) 474 allowed_projects = self._GetAllowedProjects(build_config) 475 is_replacement_allowed = self._IsReplacementAllowedFunction(build_config) 476 477 overlay_dirs = [] 478 for overlay in build_config.overlays: 479 overlay_dir = os.path.join(source_dir, 'overlays', overlay.name) 480 overlay_dirs.append(overlay_dir) 481 482 self._AddOverlays( 483 source_dir, overlay_dirs, destination_dir, 484 skip_subdirs, allowed_projects, allowed_read_write, contains_read_write, 485 is_replacement_allowed) 486 487 # If specified for this target, create a custom filesystem view 488 for path_relative_from, path_relative_to in build_config.views: 489 path_from = os.path.join(source_dir, path_relative_from) 490 if os.path.isfile(path_from) or os.path.isdir(path_from): 491 path_to = os.path.join(destination_dir, path_relative_to) 492 if allowed_read_write(path_from): 493 self._AddBindMount(path_from, path_to, False) 494 else: 495 self._AddBindMount(path_from, path_to, True) 496 else: 497 raise ValueError("Path '%s' must be a file or directory" % path_from) 498 499 self._overlay_dirs = overlay_dirs 500 if not self._quiet: 501 print('Applied overlays ' + ' '.join(self._overlay_dirs)) 502