1#!/usr/bin/env python3 2# 3# Copyright (C) 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17"""Unzips and installs the vendor snapshot.""" 18 19import argparse 20import glob 21import logging 22import os 23import re 24import shutil 25import subprocess 26import sys 27import tempfile 28import textwrap 29import json 30 31INDENT = ' ' * 4 32 33 34def get_notice_path(module_name): 35 return os.path.join('NOTICE_FILES', module_name + '.txt') 36 37 38def get_target_arch(json_rel_path): 39 return json_rel_path.split('/')[0] 40 41 42def get_arch(json_rel_path): 43 return json_rel_path.split('/')[1].split('-')[1] 44 45 46def get_variation(json_rel_path): 47 return json_rel_path.split('/')[2] 48 49# convert .bp prop dictionary to .bp prop string 50def gen_bp_prop(prop, ind): 51 bp = '' 52 for key in sorted(prop): 53 val = prop[key] 54 55 # Skip empty list or dict, rather than printing empty prop like 56 # "key: []," or "key: {}," 57 if type(val) == list and len(val) == 0: 58 continue 59 if type(val) == dict and gen_bp_prop(val, '') == '': 60 continue 61 62 bp += ind + key + ': ' 63 if type(val) == bool: 64 bp += 'true,\n' if val else 'false,\n' 65 elif type(val) == str: 66 bp += '"%s",\n' % val 67 elif type(val) == list: 68 bp += '[\n' 69 for elem in val: 70 bp += ind + INDENT + '"%s",\n' % elem 71 bp += ind + '],\n' 72 elif type(val) == dict: 73 bp += '{\n' 74 bp += gen_bp_prop(val, ind + INDENT) 75 bp += ind + '},\n' 76 else: 77 raise TypeError('unsupported type %s for gen_bp_prop' % type(val)) 78 return bp 79 80 81# Remove non-existent dirs from given list. Emits warning for such dirs. 82def remove_invalid_dirs(paths, bp_dir, module_name): 83 ret = [] 84 for path in paths: 85 if os.path.isdir(os.path.join(bp_dir, path)): 86 ret.append(path) 87 else: 88 logging.warning('Dir "%s" of module "%s" does not exist', path, 89 module_name) 90 return ret 91 92 93JSON_TO_BP = { 94 'ModuleName': 'name', 95 'ModuleStemName': 'stem', 96 'RelativeInstallPath': 'relative_install_path', 97 'ExportedDirs': 'export_include_dirs', 98 'ExportedSystemDirs': 'export_system_include_dirs', 99 'ExportedFlags': 'export_flags', 100 'Sanitize': 'sanitize', 101 'SanitizeMinimalDep': 'sanitize_minimal_dep', 102 'SanitizeUbsanDep': 'sanitize_ubsan_dep', 103 'Symlinks': 'symlinks', 104 'StaticExecutable': 'static_executable', 105 'InstallInRoot': 'install_in_root', 106 'InitRc': 'init_rc', 107 'VintfFragments': 'vintf_fragments', 108 'SharedLibs': 'shared_libs', 109 'StaticLibs': 'static_libs', 110 'RuntimeLibs': 'runtime_libs', 111 'Required': 'required', 112 'Filename': 'filename', 113 'CrateName': 'crate_name', 114 'Prebuilt': 'prebuilt', 115} 116 117SANITIZER_VARIANT_PROPS = { 118 'export_include_dirs', 119 'export_system_include_dirs', 120 'export_flags', 121 'sanitize_minimal_dep', 122 'sanitize_ubsan_dep', 123 'src', 124} 125 126EXPORTED_FLAGS_PROPS = { 127 'export_include_dirs', 128 'export_system_include_dirs', 129 'export_flags', 130} 131 132# Convert json file to Android.bp prop dictionary 133def convert_json_to_bp_prop(json_path, bp_dir): 134 prop = json.load(json_path) 135 return convert_json_data_to_bp_prop(prop, bp_dir) 136 137# Converts parsed json dictionary (which is intermediate) to Android.bp prop 138# dictionary. This validates paths such as include directories and init_rc 139# files while converting. 140def convert_json_data_to_bp_prop(prop, bp_dir): 141 ret = {} 142 143 module_name = prop['ModuleName'] 144 ret['name'] = module_name 145 146 # Soong will complain about non-existing paths on Android.bp. There might 147 # be missing files among generated header files, so check all exported 148 # directories and filter out invalid ones. Emits warning for such dirs. 149 # TODO: fix soong to track all generated headers correctly 150 for key in {'ExportedDirs', 'ExportedSystemDirs'}: 151 if key in prop: 152 prop[key] = remove_invalid_dirs(prop[key], bp_dir, module_name) 153 154 for key in prop: 155 if key in JSON_TO_BP: 156 ret[JSON_TO_BP[key]] = prop[key] 157 else: 158 logging.warning('Unknown prop "%s" of module "%s"', key, 159 module_name) 160 161 return ret 162 163def is_64bit_arch(arch): 164 return '64' in arch # arm64, x86_64 165 166def remove_keys_from_dict(keys, d): 167 # May contain subdictionaries (e.g. cfi), so recursively erase 168 for k in list(d.keys()): 169 if k in keys: 170 del d[k] 171 elif type(d[k]) == dict: 172 remove_keys_from_dict(keys, d[k]) 173 174def reexport_vndk_header(name, arch_props): 175 remove_keys_from_dict(EXPORTED_FLAGS_PROPS, arch_props) 176 for arch in arch_props: 177 arch_props[arch]['shared_libs'] = [name] 178 arch_props[arch]['export_shared_lib_headers'] = [name] 179 180def gen_bp_module(image, variation, name, version, target_arch, vndk_list, arch_props, bp_dir): 181 # Generate Android.bp module for given snapshot. 182 # If a vndk library with the same name exists, reuses exported flags of the vndk library, 183 # instead of the snapshot's own flags. 184 prop = { 185 # These are common for all snapshot modules. 186 image: True, 187 'arch': {}, 188 } 189 190 if variation != 'etc': 191 prop['version'] = str(version) 192 prop['target_arch'] = target_arch 193 else: 194 prop['prefer'] = True 195 196 reexport_vndk_name = name 197 if reexport_vndk_name == "libc++_static": 198 reexport_vndk_name = "libc++" 199 200 if reexport_vndk_name in vndk_list: 201 if variation == 'shared': 202 logging.error("Module %s is both vendor snapshot shared and vndk" % name) 203 reexport_vndk_header(reexport_vndk_name, arch_props) 204 205 # Factor out common prop among architectures to minimize Android.bp. 206 common_prop = None 207 for arch in arch_props: 208 if common_prop is None: 209 common_prop = dict() 210 for k in arch_props[arch]: 211 common_prop[k] = arch_props[arch][k] 212 continue 213 for k in list(common_prop.keys()): 214 if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]: 215 del common_prop[k] 216 217 # Some keys has to be arch_props to prevent 32-bit only modules from being 218 # used as 64-bit modules, and vice versa. 219 for arch_prop_key in ['src', 'cfi']: 220 if arch_prop_key in common_prop: 221 del common_prop[arch_prop_key] 222 prop.update(common_prop) 223 224 has32 = has64 = False 225 stem32 = stem64 = '' 226 227 for arch in arch_props: 228 for k in common_prop: 229 if k in arch_props[arch]: 230 del arch_props[arch][k] 231 prop['arch'][arch] = arch_props[arch] 232 233 has64 |= is_64bit_arch(arch) 234 has32 |= not is_64bit_arch(arch) 235 236 # Record stem for snapshots. 237 # We don't check existence of 'src'; src must exist for executables 238 if variation == 'binary': 239 if is_64bit_arch(arch): 240 stem64 = os.path.basename(arch_props[arch]['src']) 241 else: 242 stem32 = os.path.basename(arch_props[arch]['src']) 243 244 # header snapshots doesn't need compile_multilib. The other snapshots, 245 # shared/static/object/binary snapshots, do need them 246 if variation != 'header' and variation != 'etc': 247 if has32 and has64: 248 prop['compile_multilib'] = 'both' 249 elif has32: 250 prop['compile_multilib'] = '32' 251 elif has64: 252 prop['compile_multilib'] = '64' 253 else: 254 raise RuntimeError("Module %s doesn't have prebuilts." % name) 255 256 # For binary snapshots, prefer 64bit if their stem collide and installing 257 # both is impossible 258 if variation == 'binary' and stem32 == stem64: 259 prop['compile_multilib'] = 'first' 260 261 module_type = '' 262 263 if variation == 'etc': 264 module_type = 'snapshot_etc' 265 else: 266 module_type = '%s_snapshot_%s' % (image, variation) 267 268 bp = module_type + ' {\n' 269 bp += gen_bp_prop(prop, INDENT) 270 bp += '}\n\n' 271 return bp 272 273def get_vndk_list(vndk_dir, target_arch): 274 """Generates vndk_libs list, e.g. ['libbase', 'libc++', ...] 275 This list is retrieved from vndk_dir/target_arch/configs/module_names.txt. 276 If it doesn't exist, print an error message and return an empty list. 277 """ 278 279 module_names_path = os.path.join(vndk_dir, target_arch, 'configs/module_names.txt') 280 281 try: 282 with open(module_names_path, 'r') as f: 283 """The format of module_names.txt is a list of "{so_name} {module_name}", e.g. 284 285 lib1.so lib1 286 lib2.so lib2 287 ... 288 289 We extract the module name part. 290 """ 291 return [l.split()[1] for l in f.read().strip('\n').split('\n')] 292 except IOError as e: 293 logging.error('Failed to read %s: %s' % (module_names_path, e.strerror)) 294 except IndexError as e: 295 logging.error('Failed to parse %s: invalid format' % module_names_path) 296 297 return [] 298 299def gen_bp_list_module(image, snapshot_version, vndk_list, target_arch, arch_props): 300 """Generates a {image}_snapshot module which contains lists of snapshots. 301 For vendor snapshot, vndk list is also included, extracted from vndk_dir. 302 """ 303 304 bp = '%s_snapshot {\n' % image 305 306 bp_props = dict() 307 bp_props['name'] = '%s_snapshot' % image 308 bp_props['version'] = str(snapshot_version) 309 if image == 'vendor': 310 bp_props['vndk_libs'] = vndk_list 311 312 variant_to_property = { 313 'shared': 'shared_libs', 314 'static': 'static_libs', 315 'rlib': 'rlibs', 316 'header': 'header_libs', 317 'binary': 'binaries', 318 'object': 'objects', 319 } 320 321 # arch_bp_prop[arch][variant_prop] = list 322 # e.g. arch_bp_prop['x86']['shared_libs'] == ['libfoo', 'libbar', ...] 323 arch_bp_prop = dict() 324 325 # Gather module lists per arch. 326 # arch_props structure: arch_props[variant][module_name][arch] 327 # e.g. arch_props['shared']['libc++']['x86'] 328 for variant in arch_props: 329 if variant in variant_to_property: 330 variant_name = variant_to_property[variant] 331 for name in arch_props[variant]: 332 for arch in arch_props[variant][name]: 333 if arch not in arch_bp_prop: 334 arch_bp_prop[arch] = dict() 335 if variant_name not in arch_bp_prop[arch]: 336 arch_bp_prop[arch][variant_name] = [] 337 arch_bp_prop[arch][variant_name].append(name) 338 339 bp_props['arch'] = arch_bp_prop 340 bp += gen_bp_prop(bp_props, INDENT) 341 342 bp += '}\n\n' 343 return bp 344 345def build_props(install_dir): 346 # props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json 347 props = dict() 348 349 # {target_arch}/{arch}/{variation}/{module}.json 350 for root, _, files in os.walk(install_dir, followlinks = True): 351 for file_name in sorted(files): 352 if not file_name.endswith('.json'): 353 continue 354 full_path = os.path.join(root, file_name) 355 rel_path = os.path.relpath(full_path, install_dir) 356 357 target_arch = get_target_arch(rel_path) 358 arch = get_arch(rel_path) 359 variation = get_variation(rel_path) 360 bp_dir = os.path.join(install_dir, target_arch) 361 362 if not target_arch in props: 363 props[target_arch] = dict() 364 if not variation in props[target_arch]: 365 props[target_arch][variation] = dict() 366 367 with open(full_path, 'r') as f: 368 prop = convert_json_to_bp_prop(f, bp_dir) 369 # Remove .json after parsing? 370 # os.unlink(full_path) 371 372 if variation != 'header': 373 prop['src'] = os.path.relpath( 374 rel_path[:-5], # removing .json 375 target_arch) 376 377 module_name = prop['name'] 378 379 # Is this sanitized variant? 380 if 'sanitize' in prop: 381 sanitizer_type = prop['sanitize'] 382 # module_name is {name}.{sanitizer_type}; trim sanitizer_type 383 module_name = module_name[:-len(sanitizer_type) - 1] 384 # Only leave props for the sanitize variant 385 for k in list(prop.keys()): 386 if not k in SANITIZER_VARIANT_PROPS: 387 del prop[k] 388 prop = {'name': module_name, sanitizer_type: prop} 389 390 notice_path = 'NOTICE_FILES/' + module_name + '.txt' 391 if os.path.exists(os.path.join(bp_dir, notice_path)): 392 prop['notice'] = notice_path 393 394 variation_dict = props[target_arch][variation] 395 if not module_name in variation_dict: 396 variation_dict[module_name] = dict() 397 if not arch in variation_dict[module_name]: 398 variation_dict[module_name][arch] = prop 399 else: 400 variation_dict[module_name][arch].update(prop) 401 402 return props 403 404def convert_json_host_data_to_bp(mod, install_dir): 405 """Create blueprint definition for a given host module. 406 407 All host modules are created as a cc_prebuilt_binary 408 blueprint module with the prefer attribute set to true. 409 410 Modules that already have a prebuilt are not created. 411 412 Args: 413 mod: JSON definition of the module 414 install_dir: installation directory of the host snapshot 415 """ 416 rust_proc_macro = mod.pop('RustProcMacro', False) 417 prop = convert_json_data_to_bp_prop(mod, install_dir) 418 if 'prebuilt' in prop: 419 return 420 421 if not rust_proc_macro: 422 prop['host_supported'] = True 423 prop['device_supported'] = False 424 prop['stl'] = 'none' 425 426 prop['prefer'] = True 427 ## Move install file to host source file 428 prop['target'] = dict() 429 prop['target']['host'] = dict() 430 prop['target']['host']['srcs'] = [prop['filename']] 431 del prop['filename'] 432 433 mod_type = 'cc_prebuilt_binary' 434 435 if rust_proc_macro: 436 mod_type = 'rust_prebuilt_proc_macro' 437 438 bp = mod_type + ' {\n' + gen_bp_prop(prop, INDENT) + '}\n\n' 439 return bp 440 441def gen_host_bp_file(install_dir): 442 """Generate Android.bp for a host snapshot. 443 444 This routine will find the JSON description file from a host 445 snapshot and create a blueprint definition for each module 446 and add to the created Android.bp file. 447 448 Args: 449 install_dir: directory where the host snapshot can be found 450 """ 451 bpfilename = 'Android.bp' 452 with open(os.path.join(install_dir, bpfilename), 'w') as wfp: 453 for file in os.listdir(install_dir): 454 if file.endswith('.json'): 455 with open(os.path.join(install_dir, file), 'r') as rfp: 456 props = json.load(rfp) 457 for mod in props: 458 prop = convert_json_host_data_to_bp(mod, install_dir) 459 if prop: 460 wfp.write(prop) 461 462def gen_bp_files(image, vndk_dir, install_dir, snapshot_version): 463 """Generates Android.bp for each archtecture. 464 Android.bp will contain a {image}_snapshot module having lists of VNDK and 465 vendor snapshot libraries, and {image}_snapshot_{variant} modules which are 466 prebuilt libraries of the snapshot. 467 468 Args: 469 image: string, name of partition (e.g. 'vendor', 'recovery') 470 vndk_dir: string, directory to which vndk snapshot is installed 471 install_dir: string, directory to which the snapshot will be installed 472 snapshot_version: int, version of the snapshot 473 """ 474 props = build_props(install_dir) 475 476 for target_arch in sorted(props): 477 androidbp = '' 478 bp_dir = os.path.join(install_dir, target_arch) 479 vndk_list = [] 480 if image == 'vendor': 481 vndk_list = get_vndk_list(vndk_dir, target_arch) 482 483 # Generate snapshot modules. 484 for variation in sorted(props[target_arch]): 485 for name in sorted(props[target_arch][variation]): 486 androidbp += gen_bp_module(image, variation, name, 487 snapshot_version, target_arch, 488 vndk_list, 489 props[target_arch][variation][name], 490 bp_dir) 491 492 # Generate {image}_snapshot module which contains the list of modules. 493 androidbp += gen_bp_list_module(image, snapshot_version, vndk_list, 494 target_arch, props[target_arch]) 495 with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f: 496 logging.info('Generating Android.bp to: {}'.format(f.name)) 497 f.write(androidbp) 498 499 500def find_all_installed_files(install_dir): 501 installed_files = dict() 502 for root, _, files in os.walk(install_dir, followlinks = True): 503 for file_name in sorted(files): 504 if file_name.endswith('.json'): 505 continue 506 if file_name.endswith('Android.bp'): 507 continue 508 full_path = os.path.join(root, file_name) 509 size = os.stat(full_path).st_size 510 installed_files[full_path] = size 511 512 logging.debug('') 513 for f in sorted(installed_files.keys()): 514 logging.debug(f) 515 logging.debug('') 516 logging.debug('found {} installed files'.format(len(installed_files))) 517 logging.debug('') 518 return installed_files 519 520 521def find_files_in_props(target_arch, arch_install_dir, variation, name, props, file_to_info): 522 logging.debug('{} {} {} {} {}'.format( 523 target_arch, arch_install_dir, variation, name, props)) 524 525 def add_info(file, name, variation, arch, is_cfi, is_header): 526 info = (name, variation, arch, is_cfi, is_header) 527 info_list = file_to_info.get(file) 528 if not info_list: 529 info_list = [] 530 file_to_info[file] = info_list 531 info_list.append(info) 532 533 def find_file_in_list(dict, key, is_cfi): 534 list = dict.get(key) 535 logging.debug(' {} {}'.format(key, list)) 536 if list: 537 for item in list: 538 item_path = os.path.join(arch_install_dir, item) 539 add_info(item_path, name, variation, arch, is_cfi, False) 540 541 def find_file_in_dirs(dict, key, is_cfi, is_header): 542 dirs = dict.get(key) 543 logging.debug(' {} {}'.format(key, dirs)) 544 if dirs: 545 for dir in dirs: 546 dir_path = os.path.join(arch_install_dir, dir) 547 logging.debug(' scanning {}'.format(dir_path)) 548 for root, _, files in os.walk(dir_path, followlinks = True): 549 for file_name in sorted(files): 550 item_path = os.path.join(root, file_name) 551 add_info(item_path, name, variation, arch, is_cfi, is_header) 552 553 def find_file_in_dict(dict, is_cfi): 554 logging.debug(' arch {}'.format(arch)) 555 logging.debug(' name {}'.format( name)) 556 logging.debug(' is_cfi {}'.format(is_cfi)) 557 558 src = dict.get('src') 559 logging.debug(' src {}'.format(src)) 560 if src: 561 src_path = os.path.join(arch_install_dir, src) 562 add_info(src_path, name, variation, arch, is_cfi, False) 563 564 notice = dict.get('notice') 565 logging.debug(' notice {}'.format(notice)) 566 if notice: 567 notice_path = os.path.join(arch_install_dir, notice) 568 add_info(notice_path, name, variation, arch, is_cfi, False) 569 570 find_file_in_list(dict, 'init_rc', is_cfi) 571 find_file_in_list(dict, 'vintf_fragments', is_cfi) 572 573 find_file_in_dirs(dict, 'export_include_dirs', is_cfi, True) 574 find_file_in_dirs(dict, 'export_system_include_dirs', is_cfi, True) 575 576 for arch in sorted(props): 577 name = props[arch]['name'] 578 find_file_in_dict(props[arch], False) 579 cfi = props[arch].get('cfi') 580 if cfi: 581 find_file_in_dict(cfi, True) 582 583 584def find_all_props_files(install_dir): 585 586 # This function builds a database of filename to module. This means that we 587 # need to dive into the json to find the files that the vendor snapshot 588 # provides, and link these back to modules that provide them. 589 590 file_to_info = dict() 591 592 props = build_props(install_dir) 593 for target_arch in sorted(props): 594 arch_install_dir = os.path.join(install_dir, target_arch) 595 for variation in sorted(props[target_arch]): 596 for name in sorted(props[target_arch][variation]): 597 find_files_in_props( 598 target_arch, 599 arch_install_dir, 600 variation, 601 name, 602 props[target_arch][variation][name], 603 file_to_info) 604 605 logging.debug('') 606 for f in sorted(file_to_info.keys()): 607 logging.debug(f) 608 logging.debug('') 609 logging.debug('found {} props files'.format(len(file_to_info))) 610 logging.debug('') 611 return file_to_info 612 613 614def get_ninja_inputs(ninja_binary, ninja_build_file, modules): 615 """Returns the set of input file path strings for the given modules. 616 617 Uses the `ninja -t inputs` tool. 618 619 Args: 620 ninja_binary: The path to a ninja binary. 621 ninja_build_file: The path to a .ninja file from a build. 622 modules: The list of modules to scan for inputs. 623 """ 624 inputs = set() 625 cmd = [ 626 ninja_binary, 627 "-f", 628 ninja_build_file, 629 "-t", 630 "inputs", 631 "-d", 632 ] + list(modules) 633 logging.debug('invoke ninja {}'.format(cmd)) 634 inputs = inputs.union(set( 635 subprocess.check_output(cmd).decode().strip("\n").split("\n"))) 636 637 return inputs 638 639def check_host_usage(install_dir, ninja_binary, ninja_file, goals, output): 640 """Find the host modules that are in the ninja build dependency for a given goal. 641 642 To use this routine the user has installed a fake host snapshot with all 643 possible host tools into the install directory. Check the ninja build 644 graph for any mapping to the modules in this installation directory. 645 646 This routine will print out a vsdk_host_tools = [] statement with the 647 dependent tools. The user can then use the vsdk_host_tools as the 648 deps of their host_snapshot module. 649 """ 650 file_to_info = dict() 651 for file in os.listdir(install_dir): 652 if file.endswith('.json'): 653 with open(os.path.join(install_dir,file),'r') as rfp: 654 props = json.load(rfp) 655 for mod in props: 656 file_to_info[os.path.join(install_dir,mod['Filename'])] = mod['ModuleName'] 657 658 used_modules = set() 659 ninja_inputs = get_ninja_inputs(ninja_binary, ninja_file, goals) 660 ## Check for host file in ninja inputs 661 for file in file_to_info: 662 if file in ninja_inputs: 663 used_modules.add(file_to_info[file]) 664 665 with open(output, 'w') as f: 666 f.write('vsdk_host_tools = [ \n') 667 for m in sorted(used_modules): 668 f.write(' "%s",\n' % m) 669 f.write('] \n') 670 671def check_module_usage(install_dir, ninja_binary, image, ninja_file, goals, 672 output): 673 all_installed_files = find_all_installed_files(install_dir) 674 all_props_files = find_all_props_files(install_dir) 675 676 ninja_inputs = get_ninja_inputs(ninja_binary, ninja_file, goals) 677 logging.debug('') 678 logging.debug('ninja inputs') 679 for ni in ninja_inputs: 680 logging.debug(ni) 681 682 logging.debug('found {} ninja_inputs for goals {}'.format( 683 len(ninja_inputs), goals)) 684 685 # Intersect the file_to_info dict with the ninja_inputs to determine 686 # which items from the vendor snapshot are actually used by the goals. 687 688 total_size = 0 689 used_size = 0 690 used_file_to_info = dict() 691 692 for file, size in all_installed_files.items(): 693 total_size += size 694 if file in ninja_inputs: 695 logging.debug('used: {}'.format(file)) 696 used_size += size 697 info = all_props_files.get(file) 698 699 if info: 700 used_file_to_info[file] = info 701 else: 702 logging.warning('No info for file {}'.format(file)) 703 used_file_to_info[file] = 'no info' 704 705 logging.debug('Total size {}'.format(total_size)) 706 logging.debug('Used size {}'.format(used_size)) 707 logging.debug('') 708 logging.debug('used items') 709 710 used_modules = set() 711 712 for f, i in sorted(used_file_to_info.items()): 713 logging.debug('{} {}'.format(f, i)) 714 for m in i: 715 (name, variation, arch, is_cfi, is_header) = m 716 if not is_header: 717 used_modules.add(name) 718 719 with open(output, 'w') as f: 720 f.write('%s_SNAPSHOT_MODULES := \\\n' % image.upper()) 721 for m in sorted(used_modules): 722 f.write(' %s \\\n' % m) 723 724def check_call(cmd): 725 logging.debug('Running `{}`'.format(' '.join(cmd))) 726 subprocess.check_call(cmd) 727 728 729def fetch_artifact(branch, build, target, pattern, destination): 730 """Fetches build artifacts from Android Build server. 731 732 Args: 733 branch: string, branch to pull build artifacts from 734 build: string, build number to pull build artifacts from 735 target: string, target name to pull build artifacts from 736 pattern: string, pattern of build artifact file name 737 destination: string, destination to pull build artifact to 738 """ 739 fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' 740 cmd = [ 741 fetch_artifact_path, '--branch', branch, '--target', target, '--bid', 742 build, pattern, destination 743 ] 744 check_call(cmd) 745 746def install_artifacts(image, branch, build, target, local_dir, symlink, 747 install_dir): 748 """Installs vendor snapshot build artifacts to {install_dir}/v{version}. 749 750 1) Fetch build artifacts from Android Build server or from local_dir 751 2) Unzip or create symlinks to build artifacts 752 753 Args: 754 image: string, img file for which the snapshot was created (vendor, 755 recovery, etc.) 756 branch: string or None, branch name of build artifacts 757 build: string or None, build number of build artifacts 758 target: string or None, target name of build artifacts 759 local_dir: string or None, local dir to pull artifacts from 760 symlink: boolean, whether to use symlinks instead of unzipping the 761 vendor snapshot zip 762 install_dir: string, directory to install vendor snapshot 763 temp_artifact_dir: string, temp directory to hold build artifacts fetched 764 from Android Build server. For 'local' option, is set to None. 765 """ 766 artifact_pattern = image + '-*.zip' 767 768 def unzip_artifacts(artifact_dir): 769 artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) 770 for artifact in artifacts: 771 logging.info('Unzipping Vendor snapshot: {}'.format(artifact)) 772 check_call(['unzip', '-qn', artifact, '-d', install_dir]) 773 774 if branch and build and target: 775 with tempfile.TemporaryDirectory() as tmpdir: 776 logging.info( 777 'Fetching {pattern} from {branch} (bid: {build}, target: {target})' 778 .format( 779 pattern=artifact_pattern, 780 branch=branch, 781 build=build, 782 target=target)) 783 fetch_artifact(branch, build, target, artifact_pattern, tmpdir) 784 unzip_artifacts(tmpdir) 785 elif local_dir: 786 if symlink: 787 # This assumes local_dir is the location of vendor-snapshot in the 788 # build (e.g., out/soong/vendor-snapshot). 789 # 790 # Create the first level as proper directories and the next level 791 # as symlinks. 792 for item1 in os.listdir(local_dir): 793 dest_dir = os.path.join(install_dir, item1) 794 src_dir = os.path.join(local_dir, item1) 795 if os.path.isdir(src_dir): 796 check_call(['mkdir', '-p', dest_dir]) 797 # Create symlinks. 798 for item2 in os.listdir(src_dir): 799 src_item = os.path.join(src_dir, item2) 800 logging.info('Creating symlink from {} in {}'.format( 801 src_item, dest_dir)) 802 os.symlink(src_item, os.path.join(dest_dir, item2)) 803 else: 804 logging.info('Fetching local VNDK snapshot from {}'.format( 805 local_dir)) 806 unzip_artifacts(local_dir) 807 else: 808 raise RuntimeError('Neither local nor remote fetch information given.') 809 810def get_args(): 811 parser = argparse.ArgumentParser() 812 parser.add_argument( 813 'snapshot_version', 814 type=int, 815 help='Vendor snapshot version to install, e.g. "30".') 816 parser.add_argument( 817 '--image', 818 help=('Image whose snapshot is being updated (e.g., vendor, ' 819 'recovery , ramdisk, host, etc.)'), 820 default='vendor') 821 parser.add_argument('--branch', help='Branch to pull build from.') 822 parser.add_argument('--build', help='Build number to pull.') 823 parser.add_argument('--target', help='Target to pull.') 824 parser.add_argument( 825 '--local', 826 help=('Fetch local vendor snapshot artifacts from specified local ' 827 'directory instead of Android Build server. ' 828 'Example: --local /path/to/local/dir')) 829 parser.add_argument( 830 '--symlink', 831 action='store_true', 832 help='Use symlinks instead of unzipping vendor snapshot zip') 833 parser.add_argument( 834 '--install-dir', 835 required=True, 836 help=( 837 'Base directory to which vendor snapshot artifacts are installed. ' 838 'Example: --install-dir vendor/<company name>/vendor_snapshot/v30')) 839 parser.add_argument( 840 '--overwrite', 841 action='store_true', 842 help=( 843 'If provided, does not ask before overwriting the install-dir.')) 844 parser.add_argument( 845 '--check-module-usage', 846 action='store_true', 847 help='Check which modules are used.') 848 parser.add_argument( 849 '--check-module-usage-goal', 850 action='append', 851 help='Goal(s) for which --check-module-usage is calculated.') 852 parser.add_argument( 853 '--check-module-usage-ninja-file', 854 help='Ninja file for which --check-module-usage is calculated.') 855 parser.add_argument( 856 '--check-module-usage-output', 857 help='File to which to write the check-module-usage results.') 858 parser.add_argument( 859 '--vndk-dir', 860 help='Path to installed vndk snapshot directory. Needed to retrieve ' 861 'the list of VNDK. prebuilts/vndk/v{ver} will be used by default.') 862 863 parser.add_argument( 864 '-v', 865 '--verbose', 866 action='count', 867 default=0, 868 help='Increase output verbosity, e.g. "-v", "-vv".') 869 return parser.parse_args() 870 871 872def main(): 873 """Program entry point.""" 874 args = get_args() 875 876 host_image = args.image == 'host' 877 verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) 878 verbosity = min(args.verbose, 2) 879 logging.basicConfig( 880 format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', 881 level=verbose_map[verbosity]) 882 883 if not args.install_dir: 884 raise ValueError('Please provide --install-dir option.') 885 install_dir = os.path.expanduser(args.install_dir) 886 887 if args.check_module_usage: 888 ninja_binary = './prebuilts/build-tools/linux-x86/bin/ninja' 889 890 if not args.check_module_usage_goal: 891 raise ValueError('Please provide --check-module-usage-goal option.') 892 if not args.check_module_usage_ninja_file: 893 raise ValueError( 894 'Please provide --check-module-usage-ninja-file option.') 895 if not args.check_module_usage_output: 896 raise ValueError( 897 'Please provide --check-module-usage-output option.') 898 899 if host_image: 900 check_host_usage(install_dir, ninja_binary, 901 args.check_module_usage_ninja_file, 902 args.check_module_usage_goal, 903 args.check_module_usage_output) 904 else: 905 check_module_usage(install_dir, ninja_binary, args.image, 906 args.check_module_usage_ninja_file, 907 args.check_module_usage_goal, 908 args.check_module_usage_output) 909 return 910 911 local = None 912 if args.local: 913 local = os.path.expanduser(args.local) 914 915 if local: 916 if args.build or args.branch or args.target: 917 raise ValueError( 918 'When --local option is set, --branch, --build or --target cannot be ' 919 'specified.') 920 elif not os.path.isdir(local): 921 raise RuntimeError( 922 'The specified local directory, {}, does not exist.'.format( 923 local)) 924 else: 925 if not (args.build and args.branch and args.target): 926 raise ValueError( 927 'Please provide --branch, --build and --target. Or set --local ' 928 'option.') 929 930 snapshot_version = args.snapshot_version 931 932 if os.path.exists(install_dir): 933 def remove_dir(): 934 logging.info('Removing {}'.format(install_dir)) 935 check_call(['rm', '-rf', install_dir]) 936 if args.overwrite: 937 remove_dir() 938 else: 939 resp = input('Directory {} already exists. IT WILL BE REMOVED.\n' 940 'Are you sure? (yes/no): '.format(install_dir)) 941 if resp == 'yes': 942 remove_dir() 943 elif resp == 'no': 944 logging.info('Cancelled snapshot install.') 945 return 946 else: 947 raise ValueError('Did not understand: ' + resp) 948 check_call(['mkdir', '-p', install_dir]) 949 950 if args.vndk_dir: 951 vndk_dir = os.path.expanduser(args.vndk_dir) 952 else: 953 vndk_dir = 'prebuilts/vndk/v%d' % snapshot_version 954 logging.debug('Using %s for vndk directory' % vndk_dir) 955 956 install_artifacts( 957 image=args.image, 958 branch=args.branch, 959 build=args.build, 960 target=args.target, 961 local_dir=local, 962 symlink=args.symlink, 963 install_dir=install_dir) 964 965 if host_image: 966 gen_host_bp_file(install_dir) 967 968 else: 969 gen_bp_files(args.image, vndk_dir, install_dir, snapshot_version) 970 971if __name__ == '__main__': 972 main() 973