1#!/usr/bin/env python 2# 3# Copyright (C) 2011 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Builds output_image from the given input_directory, properties_file, 19and writes the image to target_output_directory. 20 21Usage: build_image input_directory properties_file output_image \\ 22 target_output_directory 23""" 24 25from __future__ import print_function 26 27import glob 28import logging 29import os 30import os.path 31import re 32import shutil 33import sys 34 35import common 36import verity_utils 37 38logger = logging.getLogger(__name__) 39 40OPTIONS = common.OPTIONS 41BLOCK_SIZE = common.BLOCK_SIZE 42BYTES_IN_MB = 1024 * 1024 43 44 45class BuildImageError(Exception): 46 """An Exception raised during image building.""" 47 48 def __init__(self, message): 49 Exception.__init__(self, message) 50 51 52def GetDiskUsage(path): 53 """Returns the number of bytes that "path" occupies on host. 54 55 Args: 56 path: The directory or file to calculate size on. 57 58 Returns: 59 The number of bytes based on a 1K block_size. 60 """ 61 cmd = ["du", "-b", "-k", "-s", path] 62 output = common.RunAndCheckOutput(cmd, verbose=False) 63 return int(output.split()[0]) * 1024 64 65 66def GetInodeUsage(path): 67 """Returns the number of inodes that "path" occupies on host. 68 69 Args: 70 path: The directory or file to calculate inode number on. 71 72 Returns: 73 The number of inodes used. 74 """ 75 cmd = ["find", path, "-print"] 76 output = common.RunAndCheckOutput(cmd, verbose=False) 77 # increase by > 6% as number of files and directories is not whole picture. 78 inodes = output.count('\n') 79 spare_inodes = inodes * 6 // 100 80 min_spare_inodes = 12 81 if spare_inodes < min_spare_inodes: 82 spare_inodes = min_spare_inodes 83 return inodes + spare_inodes 84 85 86def GetFilesystemCharacteristics(fs_type, image_path, sparse_image=True): 87 """Returns various filesystem characteristics of "image_path". 88 89 Args: 90 image_path: The file to analyze. 91 sparse_image: Image is sparse 92 93 Returns: 94 The characteristics dictionary. 95 """ 96 unsparse_image_path = image_path 97 if sparse_image: 98 unsparse_image_path = UnsparseImage(image_path, replace=False) 99 100 if fs_type.startswith("ext"): 101 cmd = ["tune2fs", "-l", unsparse_image_path] 102 elif fs_type.startswith("f2fs"): 103 cmd = ["fsck.f2fs", "-l", unsparse_image_path] 104 105 try: 106 output = common.RunAndCheckOutput(cmd, verbose=False) 107 finally: 108 if sparse_image: 109 os.remove(unsparse_image_path) 110 fs_dict = {} 111 for line in output.splitlines(): 112 fields = line.split(":") 113 if len(fields) == 2: 114 fs_dict[fields[0].strip()] = fields[1].strip() 115 return fs_dict 116 117 118def UnsparseImage(sparse_image_path, replace=True): 119 img_dir = os.path.dirname(sparse_image_path) 120 unsparse_image_path = "unsparse_" + os.path.basename(sparse_image_path) 121 unsparse_image_path = os.path.join(img_dir, unsparse_image_path) 122 if os.path.exists(unsparse_image_path): 123 if replace: 124 os.unlink(unsparse_image_path) 125 else: 126 return unsparse_image_path 127 inflate_command = ["simg2img", sparse_image_path, unsparse_image_path] 128 try: 129 common.RunAndCheckOutput(inflate_command) 130 except: 131 os.remove(unsparse_image_path) 132 raise 133 return unsparse_image_path 134 135 136def ConvertBlockMapToBaseFs(block_map_file): 137 base_fs_file = common.MakeTempFile(prefix="script_gen_", suffix=".base_fs") 138 convert_command = ["blk_alloc_to_base_fs", block_map_file, base_fs_file] 139 common.RunAndCheckOutput(convert_command) 140 return base_fs_file 141 142 143def SetUpInDirAndFsConfig(origin_in, prop_dict): 144 """Returns the in_dir and fs_config that should be used for image building. 145 146 When building system.img for all targets, it creates and returns a staged dir 147 that combines the contents of /system (i.e. in the given in_dir) and root. 148 149 Args: 150 origin_in: Path to the input directory. 151 prop_dict: A property dict that contains info like partition size. Values 152 may be updated. 153 154 Returns: 155 A tuple of in_dir and fs_config that should be used to build the image. 156 """ 157 fs_config = prop_dict.get("fs_config") 158 159 if prop_dict["mount_point"] == "system_other": 160 prop_dict["mount_point"] = "system" 161 return origin_in, fs_config 162 163 if prop_dict["mount_point"] != "system": 164 return origin_in, fs_config 165 166 if "first_pass" in prop_dict: 167 prop_dict["mount_point"] = "/" 168 return prop_dict["first_pass"] 169 170 # Construct a staging directory of the root file system. 171 in_dir = common.MakeTempDir() 172 root_dir = prop_dict.get("root_dir") 173 if root_dir: 174 shutil.rmtree(in_dir) 175 shutil.copytree(root_dir, in_dir, symlinks=True) 176 in_dir_system = os.path.join(in_dir, "system") 177 shutil.rmtree(in_dir_system, ignore_errors=True) 178 shutil.copytree(origin_in, in_dir_system, symlinks=True) 179 180 # Change the mount point to "/". 181 prop_dict["mount_point"] = "/" 182 if fs_config: 183 # We need to merge the fs_config files of system and root. 184 merged_fs_config = common.MakeTempFile( 185 prefix="merged_fs_config", suffix=".txt") 186 with open(merged_fs_config, "w") as fw: 187 if "root_fs_config" in prop_dict: 188 with open(prop_dict["root_fs_config"]) as fr: 189 fw.writelines(fr.readlines()) 190 with open(fs_config) as fr: 191 fw.writelines(fr.readlines()) 192 fs_config = merged_fs_config 193 prop_dict["first_pass"] = (in_dir, fs_config) 194 return in_dir, fs_config 195 196 197def CheckHeadroom(ext4fs_output, prop_dict): 198 """Checks if there's enough headroom space available. 199 200 Headroom is the reserved space on system image (via PRODUCT_SYSTEM_HEADROOM), 201 which is useful for devices with low disk space that have system image 202 variation between builds. The 'partition_headroom' in prop_dict is the size 203 in bytes, while the numbers in 'ext4fs_output' are for 4K-blocks. 204 205 Args: 206 ext4fs_output: The output string from mke2fs command. 207 prop_dict: The property dict. 208 209 Raises: 210 AssertionError: On invalid input. 211 BuildImageError: On check failure. 212 """ 213 assert ext4fs_output is not None 214 assert prop_dict.get('fs_type', '').startswith('ext4') 215 assert 'partition_headroom' in prop_dict 216 assert 'mount_point' in prop_dict 217 218 ext4fs_stats = re.compile( 219 r'Created filesystem with .* (?P<used_blocks>[0-9]+)/' 220 r'(?P<total_blocks>[0-9]+) blocks') 221 last_line = ext4fs_output.strip().split('\n')[-1] 222 m = ext4fs_stats.match(last_line) 223 used_blocks = int(m.groupdict().get('used_blocks')) 224 total_blocks = int(m.groupdict().get('total_blocks')) 225 headroom_blocks = int(prop_dict['partition_headroom']) // BLOCK_SIZE 226 adjusted_blocks = total_blocks - headroom_blocks 227 if used_blocks > adjusted_blocks: 228 mount_point = prop_dict["mount_point"] 229 raise BuildImageError( 230 "Error: Not enough room on {} (total: {} blocks, used: {} blocks, " 231 "headroom: {} blocks, available: {} blocks)".format( 232 mount_point, total_blocks, used_blocks, headroom_blocks, 233 adjusted_blocks)) 234 235 236def CalculateSizeAndReserved(prop_dict, size): 237 fs_type = prop_dict.get("fs_type", "") 238 partition_headroom = int(prop_dict.get("partition_headroom", 0)) 239 # If not specified, give us 16MB margin for GetDiskUsage error ... 240 reserved_size = int(prop_dict.get( 241 "partition_reserved_size", BYTES_IN_MB * 16)) 242 243 if fs_type == "erofs": 244 reserved_size = int(prop_dict.get("partition_reserved_size", 0)) 245 if reserved_size == 0: 246 # give .3% margin or a minimum size for AVB footer 247 return max(size * 1003 // 1000, 256 * 1024) 248 249 if fs_type.startswith("ext4") and partition_headroom > reserved_size: 250 reserved_size = partition_headroom 251 252 return size + reserved_size 253 254 255def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config): 256 """Builds a pure image for the files under in_dir and writes it to out_file. 257 258 Args: 259 in_dir: Path to input directory. 260 prop_dict: A property dict that contains info like partition size. Values 261 will be updated with computed values. 262 out_file: The output image file. 263 target_out: Path to the TARGET_OUT directory as in Makefile. It actually 264 points to the /system directory under PRODUCT_OUT. fs_config (the one 265 under system/core/libcutils) reads device specific FS config files from 266 there. 267 fs_config: The fs_config file that drives the prototype 268 269 Raises: 270 BuildImageError: On build image failures. 271 """ 272 build_command = [] 273 fs_type = prop_dict.get("fs_type", "") 274 run_fsck = None 275 needs_projid = prop_dict.get("needs_projid", 0) 276 needs_casefold = prop_dict.get("needs_casefold", 0) 277 needs_compress = prop_dict.get("needs_compress", 0) 278 279 disable_sparse = "disable_sparse" in prop_dict 280 manual_sparse = False 281 282 if fs_type.startswith("ext"): 283 build_command = [prop_dict["ext_mkuserimg"]] 284 if "extfs_sparse_flag" in prop_dict and not disable_sparse: 285 build_command.append(prop_dict["extfs_sparse_flag"]) 286 run_e2fsck = RunE2fsck 287 build_command.extend([in_dir, out_file, fs_type, 288 prop_dict["mount_point"]]) 289 build_command.append(prop_dict["image_size"]) 290 if "journal_size" in prop_dict: 291 build_command.extend(["-j", prop_dict["journal_size"]]) 292 if "timestamp" in prop_dict: 293 build_command.extend(["-T", str(prop_dict["timestamp"])]) 294 if fs_config: 295 build_command.extend(["-C", fs_config]) 296 if target_out: 297 build_command.extend(["-D", target_out]) 298 if "block_list" in prop_dict: 299 build_command.extend(["-B", prop_dict["block_list"]]) 300 if "base_fs_file" in prop_dict: 301 base_fs_file = ConvertBlockMapToBaseFs(prop_dict["base_fs_file"]) 302 build_command.extend(["-d", base_fs_file]) 303 build_command.extend(["-L", prop_dict["mount_point"]]) 304 if "extfs_inode_count" in prop_dict: 305 build_command.extend(["-i", prop_dict["extfs_inode_count"]]) 306 if "extfs_rsv_pct" in prop_dict: 307 build_command.extend(["-M", prop_dict["extfs_rsv_pct"]]) 308 if "flash_erase_block_size" in prop_dict: 309 build_command.extend(["-e", prop_dict["flash_erase_block_size"]]) 310 if "flash_logical_block_size" in prop_dict: 311 build_command.extend(["-o", prop_dict["flash_logical_block_size"]]) 312 # Specify UUID and hash_seed if using mke2fs. 313 if os.path.basename(prop_dict["ext_mkuserimg"]) == "mkuserimg_mke2fs": 314 if "uuid" in prop_dict: 315 build_command.extend(["-U", prop_dict["uuid"]]) 316 if "hash_seed" in prop_dict: 317 build_command.extend(["-S", prop_dict["hash_seed"]]) 318 if prop_dict.get("ext4_share_dup_blocks") == "true": 319 build_command.append("-c") 320 if (needs_projid): 321 build_command.extend(["--inode_size", "512"]) 322 else: 323 build_command.extend(["--inode_size", "256"]) 324 if "selinux_fc" in prop_dict: 325 build_command.append(prop_dict["selinux_fc"]) 326 elif fs_type.startswith("erofs"): 327 build_command = ["mkfs.erofs"] 328 329 compressor = None 330 if "erofs_default_compressor" in prop_dict: 331 compressor = prop_dict["erofs_default_compressor"] 332 if "erofs_compressor" in prop_dict: 333 compressor = prop_dict["erofs_compressor"] 334 if compressor and compressor != "none": 335 build_command.extend(["-z", compressor]) 336 337 compress_hints = None 338 if "erofs_default_compress_hints" in prop_dict: 339 compress_hints = prop_dict["erofs_default_compress_hints"] 340 if "erofs_compress_hints" in prop_dict: 341 compress_hints = prop_dict["erofs_compress_hints"] 342 if compress_hints: 343 build_command.extend(["--compress-hints", compress_hints]) 344 345 build_command.extend(["--mount-point", prop_dict["mount_point"]]) 346 if target_out: 347 build_command.extend(["--product-out", target_out]) 348 if fs_config: 349 build_command.extend(["--fs-config-file", fs_config]) 350 if "selinux_fc" in prop_dict: 351 build_command.extend(["--file-contexts", prop_dict["selinux_fc"]]) 352 if "timestamp" in prop_dict: 353 build_command.extend(["-T", str(prop_dict["timestamp"])]) 354 if "uuid" in prop_dict: 355 build_command.extend(["-U", prop_dict["uuid"]]) 356 if "block_list" in prop_dict: 357 build_command.extend(["--block-list-file", prop_dict["block_list"]]) 358 if "erofs_pcluster_size" in prop_dict: 359 build_command.extend(["-C", prop_dict["erofs_pcluster_size"]]) 360 if "erofs_share_dup_blocks" in prop_dict: 361 build_command.extend(["--chunksize", "4096"]) 362 if "erofs_use_legacy_compression" in prop_dict: 363 build_command.extend(["-E", "legacy-compress"]) 364 365 build_command.extend([out_file, in_dir]) 366 if "erofs_sparse_flag" in prop_dict and not disable_sparse: 367 manual_sparse = True 368 369 run_fsck = RunErofsFsck 370 elif fs_type.startswith("squash"): 371 build_command = ["mksquashfsimage"] 372 build_command.extend([in_dir, out_file]) 373 if "squashfs_sparse_flag" in prop_dict and not disable_sparse: 374 build_command.extend([prop_dict["squashfs_sparse_flag"]]) 375 build_command.extend(["-m", prop_dict["mount_point"]]) 376 if target_out: 377 build_command.extend(["-d", target_out]) 378 if fs_config: 379 build_command.extend(["-C", fs_config]) 380 if "selinux_fc" in prop_dict: 381 build_command.extend(["-c", prop_dict["selinux_fc"]]) 382 if "block_list" in prop_dict: 383 build_command.extend(["-B", prop_dict["block_list"]]) 384 if "squashfs_block_size" in prop_dict: 385 build_command.extend(["-b", prop_dict["squashfs_block_size"]]) 386 if "squashfs_compressor" in prop_dict: 387 build_command.extend(["-z", prop_dict["squashfs_compressor"]]) 388 if "squashfs_compressor_opt" in prop_dict: 389 build_command.extend(["-zo", prop_dict["squashfs_compressor_opt"]]) 390 if prop_dict.get("squashfs_disable_4k_align") == "true": 391 build_command.extend(["-a"]) 392 elif fs_type.startswith("f2fs"): 393 build_command = ["mkf2fsuserimg"] 394 build_command.extend([out_file, prop_dict["image_size"]]) 395 if "f2fs_sparse_flag" in prop_dict and not disable_sparse: 396 build_command.extend([prop_dict["f2fs_sparse_flag"]]) 397 if fs_config: 398 build_command.extend(["-C", fs_config]) 399 build_command.extend(["-f", in_dir]) 400 if target_out: 401 build_command.extend(["-D", target_out]) 402 if "selinux_fc" in prop_dict: 403 build_command.extend(["-s", prop_dict["selinux_fc"]]) 404 build_command.extend(["-t", prop_dict["mount_point"]]) 405 if "timestamp" in prop_dict: 406 build_command.extend(["-T", str(prop_dict["timestamp"])]) 407 if "block_list" in prop_dict: 408 build_command.extend(["-B", prop_dict["block_list"]]) 409 build_command.extend(["-L", prop_dict["mount_point"]]) 410 if (needs_projid): 411 build_command.append("--prjquota") 412 if (needs_casefold): 413 build_command.append("--casefold") 414 if (needs_compress or prop_dict.get("f2fs_compress") == "true"): 415 build_command.append("--compression") 416 if "ro_mount_point" in prop_dict: 417 build_command.append("--readonly") 418 if (prop_dict.get("f2fs_compress") == "true"): 419 build_command.append("--sldc") 420 if (prop_dict.get("f2fs_sldc_flags") == None): 421 build_command.append(str(0)) 422 else: 423 sldc_flags_str = prop_dict.get("f2fs_sldc_flags") 424 sldc_flags = sldc_flags_str.split() 425 build_command.append(str(len(sldc_flags))) 426 build_command.extend(sldc_flags) 427 else: 428 raise BuildImageError( 429 "Error: unknown filesystem type: {}".format(fs_type)) 430 431 try: 432 mkfs_output = common.RunAndCheckOutput(build_command) 433 except: 434 try: 435 du = GetDiskUsage(in_dir) 436 du_str = "{} bytes ({} MB)".format(du, du // BYTES_IN_MB) 437 # Suppress any errors from GetDiskUsage() to avoid hiding the real errors 438 # from common.RunAndCheckOutput(). 439 except Exception: # pylint: disable=broad-except 440 logger.exception("Failed to compute disk usage with du") 441 du_str = "unknown" 442 print( 443 "Out of space? Out of inodes? The tree size of {} is {}, " 444 "with reserved space of {} bytes ({} MB).".format( 445 in_dir, du_str, 446 int(prop_dict.get("partition_reserved_size", 0)), 447 int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB)) 448 if ("image_size" in prop_dict and "partition_size" in prop_dict): 449 print( 450 "The max image size for filesystem files is {} bytes ({} MB), " 451 "out of a total partition size of {} bytes ({} MB).".format( 452 int(prop_dict["image_size"]), 453 int(prop_dict["image_size"]) // BYTES_IN_MB, 454 int(prop_dict["partition_size"]), 455 int(prop_dict["partition_size"]) // BYTES_IN_MB)) 456 raise 457 458 if run_fsck and prop_dict.get("skip_fsck") != "true": 459 run_fsck(out_file) 460 461 if manual_sparse: 462 temp_file = out_file + ".sparse" 463 img2simg_argv = ["img2simg", out_file, temp_file] 464 common.RunAndCheckOutput(img2simg_argv) 465 os.rename(temp_file, out_file) 466 467 return mkfs_output 468 469 470def RunE2fsck(out_file): 471 unsparse_image = UnsparseImage(out_file, replace=False) 472 473 # Run e2fsck on the inflated image file 474 e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image] 475 try: 476 common.RunAndCheckOutput(e2fsck_command) 477 finally: 478 os.remove(unsparse_image) 479 480 481def RunErofsFsck(out_file): 482 fsck_command = ["fsck.erofs", "--extract", out_file] 483 try: 484 common.RunAndCheckOutput(fsck_command) 485 except: 486 print("Check failed for EROFS image {}".format(out_file)) 487 raise 488 489 490def BuildImage(in_dir, prop_dict, out_file, target_out=None): 491 """Builds an image for the files under in_dir and writes it to out_file. 492 493 Args: 494 in_dir: Path to input directory. 495 prop_dict: A property dict that contains info like partition size. Values 496 will be updated with computed values. 497 out_file: The output image file. 498 target_out: Path to the TARGET_OUT directory as in Makefile. It actually 499 points to the /system directory under PRODUCT_OUT. fs_config (the one 500 under system/core/libcutils) reads device specific FS config files from 501 there. 502 503 Raises: 504 BuildImageError: On build image failures. 505 """ 506 in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict) 507 508 build_command = [] 509 fs_type = prop_dict.get("fs_type", "") 510 511 fs_spans_partition = True 512 if fs_type.startswith("squash") or fs_type.startswith("erofs"): 513 fs_spans_partition = False 514 elif fs_type.startswith("f2fs") and prop_dict.get("f2fs_compress") == "true": 515 fs_spans_partition = False 516 517 # Get a builder for creating an image that's to be verified by Verified Boot, 518 # or None if not applicable. 519 verity_image_builder = verity_utils.CreateVerityImageBuilder(prop_dict) 520 521 disable_sparse = "disable_sparse" in prop_dict 522 mkfs_output = None 523 if (prop_dict.get("use_dynamic_partition_size") == "true" and 524 "partition_size" not in prop_dict): 525 # If partition_size is not defined, use output of `du' + reserved_size. 526 # For compressed file system, it's better to use the compressed size to avoid wasting space. 527 if fs_type.startswith("erofs"): 528 mkfs_output = BuildImageMkfs( 529 in_dir, prop_dict, out_file, target_out, fs_config) 530 if "erofs_sparse_flag" in prop_dict and not disable_sparse: 531 image_path = UnsparseImage(out_file, replace=False) 532 size = GetDiskUsage(image_path) 533 os.remove(image_path) 534 else: 535 size = GetDiskUsage(out_file) 536 else: 537 size = GetDiskUsage(in_dir) 538 logger.info( 539 "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB) 540 size = CalculateSizeAndReserved(prop_dict, size) 541 # Round this up to a multiple of 4K so that avbtool works 542 size = common.RoundUpTo4K(size) 543 if fs_type.startswith("ext"): 544 prop_dict["partition_size"] = str(size) 545 prop_dict["image_size"] = str(size) 546 if "extfs_inode_count" not in prop_dict: 547 prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir)) 548 logger.info( 549 "First Pass based on estimates of %d MB and %s inodes.", 550 size // BYTES_IN_MB, prop_dict["extfs_inode_count"]) 551 BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config) 552 sparse_image = False 553 if "extfs_sparse_flag" in prop_dict and not disable_sparse: 554 sparse_image = True 555 fs_dict = GetFilesystemCharacteristics(fs_type, out_file, sparse_image) 556 os.remove(out_file) 557 block_size = int(fs_dict.get("Block size", "4096")) 558 free_size = int(fs_dict.get("Free blocks", "0")) * block_size 559 reserved_size = int(prop_dict.get("partition_reserved_size", 0)) 560 partition_headroom = int(fs_dict.get("partition_headroom", 0)) 561 if fs_type.startswith("ext4") and partition_headroom > reserved_size: 562 reserved_size = partition_headroom 563 if free_size <= reserved_size: 564 logger.info( 565 "Not worth reducing image %d <= %d.", free_size, reserved_size) 566 else: 567 size -= free_size 568 size += reserved_size 569 if reserved_size == 0: 570 # add .3% margin 571 size = size * 1003 // 1000 572 # Use a minimum size, otherwise we will fail to calculate an AVB footer 573 # or fail to construct an ext4 image. 574 size = max(size, 256 * 1024) 575 if block_size <= 4096: 576 size = common.RoundUpTo4K(size) 577 else: 578 size = ((size + block_size - 1) // block_size) * block_size 579 extfs_inode_count = prop_dict["extfs_inode_count"] 580 inodes = int(fs_dict.get("Inode count", extfs_inode_count)) 581 inodes -= int(fs_dict.get("Free inodes", "0")) 582 # add .2% margin or 1 inode, whichever is greater 583 spare_inodes = inodes * 2 // 1000 584 min_spare_inodes = 1 585 if spare_inodes < min_spare_inodes: 586 spare_inodes = min_spare_inodes 587 inodes += spare_inodes 588 prop_dict["extfs_inode_count"] = str(inodes) 589 prop_dict["partition_size"] = str(size) 590 logger.info( 591 "Allocating %d Inodes for %s.", inodes, out_file) 592 elif fs_type.startswith("f2fs") and prop_dict.get("f2fs_compress") == "true": 593 prop_dict["partition_size"] = str(size) 594 prop_dict["image_size"] = str(size) 595 BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config) 596 sparse_image = False 597 if "f2fs_sparse_flag" in prop_dict and not disable_sparse: 598 sparse_image = True 599 fs_dict = GetFilesystemCharacteristics(fs_type, out_file, sparse_image) 600 os.remove(out_file) 601 block_count = int(fs_dict.get("block_count", "0")) 602 log_blocksize = int(fs_dict.get("log_blocksize", "12")) 603 size = block_count << log_blocksize 604 prop_dict["partition_size"] = str(size) 605 if verity_image_builder: 606 size = verity_image_builder.CalculateDynamicPartitionSize(size) 607 prop_dict["partition_size"] = str(size) 608 logger.info( 609 "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file) 610 611 prop_dict["image_size"] = prop_dict["partition_size"] 612 613 # Adjust the image size to make room for the hashes if this is to be verified. 614 if verity_image_builder: 615 max_image_size = verity_image_builder.CalculateMaxImageSize() 616 prop_dict["image_size"] = str(max_image_size) 617 618 if not mkfs_output: 619 mkfs_output = BuildImageMkfs( 620 in_dir, prop_dict, out_file, target_out, fs_config) 621 622 # Update the image (eg filesystem size). This can be different eg if mkfs 623 # rounds the requested size down due to alignment. 624 prop_dict["image_size"] = common.sparse_img.GetImagePartitionSize(out_file) 625 626 # Check if there's enough headroom space available for ext4 image. 627 if "partition_headroom" in prop_dict and fs_type.startswith("ext4"): 628 CheckHeadroom(mkfs_output, prop_dict) 629 630 if not fs_spans_partition and verity_image_builder: 631 verity_image_builder.PadSparseImage(out_file) 632 633 # Create the verified image if this is to be verified. 634 if verity_image_builder: 635 verity_image_builder.Build(out_file) 636 637 638def ImagePropFromGlobalDict(glob_dict, mount_point): 639 """Build an image property dictionary from the global dictionary. 640 641 Args: 642 glob_dict: the global dictionary from the build system. 643 mount_point: such as "system", "data" etc. 644 """ 645 d = {} 646 647 if "build.prop" in glob_dict: 648 timestamp = glob_dict["build.prop"].GetProp("ro.build.date.utc") 649 if timestamp: 650 d["timestamp"] = timestamp 651 652 def copy_prop(src_p, dest_p): 653 """Copy a property from the global dictionary. 654 655 Args: 656 src_p: The source property in the global dictionary. 657 dest_p: The destination property. 658 Returns: 659 True if property was found and copied, False otherwise. 660 """ 661 if src_p in glob_dict: 662 d[dest_p] = str(glob_dict[src_p]) 663 return True 664 return False 665 666 common_props = ( 667 "extfs_sparse_flag", 668 "erofs_default_compressor", 669 "erofs_default_compress_hints", 670 "erofs_pcluster_size", 671 "erofs_share_dup_blocks", 672 "erofs_sparse_flag", 673 "erofs_use_legacy_compression", 674 "squashfs_sparse_flag", 675 "system_f2fs_compress", 676 "system_f2fs_sldc_flags", 677 "f2fs_sparse_flag", 678 "skip_fsck", 679 "ext_mkuserimg", 680 "avb_enable", 681 "avb_avbtool", 682 "use_dynamic_partition_size", 683 ) 684 for p in common_props: 685 copy_prop(p, p) 686 687 ro_mount_points = set([ 688 "odm", 689 "odm_dlkm", 690 "oem", 691 "product", 692 "system", 693 "system_dlkm", 694 "system_ext", 695 "system_other", 696 "vendor", 697 "vendor_dlkm", 698 ]) 699 700 # Tuple layout: (readonly, specific prop, general prop) 701 fmt_props = ( 702 # Generic first, then specific file type. 703 (False, "fs_type", "fs_type"), 704 (False, "{}_fs_type", "fs_type"), 705 706 # Ordering for these doesn't matter. 707 (False, "{}_selinux_fc", "selinux_fc"), 708 (False, "{}_size", "partition_size"), 709 (True, "avb_{}_add_hashtree_footer_args", "avb_add_hashtree_footer_args"), 710 (True, "avb_{}_algorithm", "avb_algorithm"), 711 (True, "avb_{}_hashtree_enable", "avb_hashtree_enable"), 712 (True, "avb_{}_key_path", "avb_key_path"), 713 (True, "avb_{}_salt", "avb_salt"), 714 (True, "erofs_use_legacy_compression", "erofs_use_legacy_compression"), 715 (True, "ext4_share_dup_blocks", "ext4_share_dup_blocks"), 716 (True, "{}_base_fs_file", "base_fs_file"), 717 (True, "{}_disable_sparse", "disable_sparse"), 718 (True, "{}_erofs_compressor", "erofs_compressor"), 719 (True, "{}_erofs_compress_hints", "erofs_compress_hints"), 720 (True, "{}_erofs_pcluster_size", "erofs_pcluster_size"), 721 (True, "{}_erofs_share_dup_blocks", "erofs_share_dup_blocks"), 722 (True, "{}_extfs_inode_count", "extfs_inode_count"), 723 (True, "{}_f2fs_compress", "f2fs_compress"), 724 (True, "{}_f2fs_sldc_flags", "f2fs_sldc_flags"), 725 (True, "{}_reserved_size", "partition_reserved_size"), 726 (True, "{}_squashfs_block_size", "squashfs_block_size"), 727 (True, "{}_squashfs_compressor", "squashfs_compressor"), 728 (True, "{}_squashfs_compressor_opt", "squashfs_compressor_opt"), 729 (True, "{}_squashfs_disable_4k_align", "squashfs_disable_4k_align"), 730 (True, "{}_verity_block_device", "verity_block_device"), 731 ) 732 733 # Translate prefixed properties into generic ones. 734 if mount_point == "data": 735 prefix = "userdata" 736 else: 737 prefix = mount_point 738 739 for readonly, src_prop, dest_prop in fmt_props: 740 if readonly and mount_point not in ro_mount_points: 741 continue 742 743 if src_prop == "fs_type": 744 # This property is legacy and only used on a few partitions. b/202600377 745 allowed_partitions = set(["system", "system_other", "data", "oem"]) 746 if mount_point not in allowed_partitions: 747 continue 748 749 if (mount_point == "system_other") and (dest_prop != "partition_size"): 750 # Propagate system properties to system_other. They'll get overridden 751 # after as needed. 752 copy_prop(src_prop.format("system"), dest_prop) 753 754 copy_prop(src_prop.format(prefix), dest_prop) 755 756 # Set prefixed properties that need a default value. 757 if mount_point in ro_mount_points: 758 prop = "{}_journal_size".format(prefix) 759 if not copy_prop(prop, "journal_size"): 760 d["journal_size"] = "0" 761 762 prop = "{}_extfs_rsv_pct".format(prefix) 763 if not copy_prop(prop, "extfs_rsv_pct"): 764 d["extfs_rsv_pct"] = "0" 765 766 d["ro_mount_point"] = "1" 767 768 # Copy partition-specific properties. 769 d["mount_point"] = mount_point 770 if mount_point == "system": 771 copy_prop("system_headroom", "partition_headroom") 772 copy_prop("system_root_image", "system_root_image") 773 copy_prop("root_dir", "root_dir") 774 copy_prop("root_fs_config", "root_fs_config") 775 elif mount_point == "data": 776 # Copy the generic fs type first, override with specific one if available. 777 copy_prop("flash_logical_block_size", "flash_logical_block_size") 778 copy_prop("flash_erase_block_size", "flash_erase_block_size") 779 copy_prop("needs_casefold", "needs_casefold") 780 copy_prop("needs_projid", "needs_projid") 781 copy_prop("needs_compress", "needs_compress") 782 d["partition_name"] = mount_point 783 return d 784 785 786def LoadGlobalDict(filename): 787 """Load "name=value" pairs from filename""" 788 d = {} 789 f = open(filename) 790 for line in f: 791 line = line.strip() 792 if not line or line.startswith("#"): 793 continue 794 k, v = line.split("=", 1) 795 d[k] = v 796 f.close() 797 return d 798 799 800def GlobalDictFromImageProp(image_prop, mount_point): 801 d = {} 802 803 def copy_prop(src_p, dest_p): 804 if src_p in image_prop: 805 d[dest_p] = image_prop[src_p] 806 return True 807 return False 808 809 if mount_point == "system": 810 copy_prop("partition_size", "system_size") 811 elif mount_point == "system_other": 812 copy_prop("partition_size", "system_other_size") 813 elif mount_point == "vendor": 814 copy_prop("partition_size", "vendor_size") 815 elif mount_point == "odm": 816 copy_prop("partition_size", "odm_size") 817 elif mount_point == "vendor_dlkm": 818 copy_prop("partition_size", "vendor_dlkm_size") 819 elif mount_point == "odm_dlkm": 820 copy_prop("partition_size", "odm_dlkm_size") 821 elif mount_point == "system_dlkm": 822 copy_prop("partition_size", "system_dlkm_size") 823 elif mount_point == "product": 824 copy_prop("partition_size", "product_size") 825 elif mount_point == "system_ext": 826 copy_prop("partition_size", "system_ext_size") 827 return d 828 829 830def BuildVBMeta(in_dir, glob_dict, output_path): 831 """Creates a VBMeta image. 832 833 It generates the requested VBMeta image. The requested image could be for 834 top-level or chained VBMeta image, which is determined based on the name. 835 836 Args: 837 output_path: Path to generated vbmeta.img 838 partitions: A dict that's keyed by partition names with image paths as 839 values. Only valid partition names are accepted, as partitions listed 840 in common.AVB_PARTITIONS and custom partitions listed in 841 OPTIONS.info_dict.get("avb_custom_images_partition_list") 842 name: Name of the VBMeta partition, e.g. 'vbmeta', 'vbmeta_system'. 843 needed_partitions: Partitions whose descriptors should be included into the 844 generated VBMeta image. 845 846 Returns: 847 Path to the created image. 848 849 Raises: 850 AssertionError: On invalid input args. 851 """ 852 vbmeta_partitions = common.AVB_PARTITIONS[:] 853 name = os.path.basename(output_path).rstrip(".img") 854 vbmeta_system = glob_dict.get("avb_vbmeta_system", "").strip() 855 vbmeta_vendor = glob_dict.get("avb_vbmeta_vendor", "").strip() 856 if "vbmeta_system" in name: 857 vbmeta_partitions = vbmeta_system.split() 858 elif "vbmeta_vendor" in name: 859 vbmeta_partitions = vbmeta_vendor.split() 860 else: 861 if vbmeta_system: 862 vbmeta_partitions = [ 863 item for item in vbmeta_partitions 864 if item not in vbmeta_system.split()] 865 vbmeta_partitions.append("vbmeta_system") 866 867 if vbmeta_vendor: 868 vbmeta_partitions = [ 869 item for item in vbmeta_partitions 870 if item not in vbmeta_vendor.split()] 871 vbmeta_partitions.append("vbmeta_vendor") 872 873 874 partitions = {part: os.path.join(in_dir, part + ".img") 875 for part in vbmeta_partitions} 876 partitions = {part:path for (part, path) in partitions.items() if os.path.exists(path)} 877 common.BuildVBMeta(output_path, partitions, name, vbmeta_partitions) 878 879 880def main(argv): 881 args = common.ParseOptions(argv, __doc__) 882 883 if len(args) != 4: 884 print(__doc__) 885 sys.exit(1) 886 887 common.InitLogging() 888 889 in_dir = args[0] 890 glob_dict_file = args[1] 891 out_file = args[2] 892 target_out = args[3] 893 894 glob_dict = LoadGlobalDict(glob_dict_file) 895 if "mount_point" in glob_dict: 896 # The caller knows the mount point and provides a dictionary needed by 897 # BuildImage(). 898 image_properties = glob_dict 899 else: 900 image_filename = os.path.basename(out_file) 901 mount_point = "" 902 if image_filename == "system.img": 903 mount_point = "system" 904 elif image_filename == "system_other.img": 905 mount_point = "system_other" 906 elif image_filename == "userdata.img": 907 mount_point = "data" 908 elif image_filename == "cache.img": 909 mount_point = "cache" 910 elif image_filename == "vendor.img": 911 mount_point = "vendor" 912 elif image_filename == "odm.img": 913 mount_point = "odm" 914 elif image_filename == "vendor_dlkm.img": 915 mount_point = "vendor_dlkm" 916 elif image_filename == "odm_dlkm.img": 917 mount_point = "odm_dlkm" 918 elif image_filename == "system_dlkm.img": 919 mount_point = "system_dlkm" 920 elif image_filename == "oem.img": 921 mount_point = "oem" 922 elif image_filename == "product.img": 923 mount_point = "product" 924 elif image_filename == "system_ext.img": 925 mount_point = "system_ext" 926 elif "vbmeta" in image_filename: 927 mount_point = "vbmeta" 928 else: 929 logger.error("Unknown image file name %s", image_filename) 930 sys.exit(1) 931 932 if "vbmeta" != mount_point: 933 image_properties = ImagePropFromGlobalDict(glob_dict, mount_point) 934 935 try: 936 if "vbmeta" in os.path.basename(out_file): 937 OPTIONS.info_dict = glob_dict 938 BuildVBMeta(in_dir, glob_dict, out_file) 939 else: 940 BuildImage(in_dir, image_properties, out_file, target_out) 941 except: 942 logger.error("Failed to build %s from %s", out_file, in_dir) 943 raise 944 945 946if __name__ == '__main__': 947 try: 948 main(sys.argv[1:]) 949 finally: 950 common.Cleanup() 951