1#!/usr/bin/env python3 2# 3# Copyright (C) 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17"""Unzips and installs the vendor snapshot.""" 18 19import argparse 20import glob 21import logging 22import os 23import re 24import shutil 25import subprocess 26import sys 27import tempfile 28import textwrap 29import json 30 31INDENT = ' ' * 4 32 33 34def get_target_arch(json_rel_path): 35 return json_rel_path.split('/')[0] 36 37 38def get_arch(json_rel_path): 39 return json_rel_path.split('/')[1].split('-')[1] 40 41 42def get_variation(json_rel_path): 43 return json_rel_path.split('/')[2] 44 45# convert .bp prop dictionary to .bp prop string 46def gen_bp_prop(prop, ind): 47 bp = '' 48 for key in sorted(prop): 49 val = prop[key] 50 51 # Skip empty list or dict, rather than printing empty prop like 52 # "key: []," or "key: {}," 53 if type(val) == list and len(val) == 0: 54 continue 55 if type(val) == dict and gen_bp_prop(val, '') == '': 56 continue 57 58 bp += ind + key + ': ' 59 if type(val) == bool: 60 bp += 'true,\n' if val else 'false,\n' 61 elif type(val) == str: 62 bp += '"%s",\n' % val 63 elif type(val) == list: 64 bp += '[\n' 65 for elem in val: 66 bp += ind + INDENT + '"%s",\n' % elem 67 bp += ind + '],\n' 68 elif type(val) == dict: 69 bp += '{\n' 70 bp += gen_bp_prop(val, ind + INDENT) 71 bp += ind + '},\n' 72 else: 73 raise TypeError('unsupported type %s for gen_bp_prop' % type(val)) 74 return bp 75 76 77# Remove non-existent dirs from given list. Emits warning for such dirs. 78def remove_invalid_dirs(paths, bp_dir, module_name): 79 ret = [] 80 for path in paths: 81 if os.path.isdir(os.path.join(bp_dir, path)): 82 ret.append(path) 83 else: 84 logging.warning('Dir "%s" of module "%s" does not exist', path, 85 module_name) 86 return ret 87 88 89JSON_TO_BP = { 90 'ModuleName': 'name', 91 'ModuleStemName': 'stem', 92 'RelativeInstallPath': 'relative_install_path', 93 'ExportedDirs': 'export_include_dirs', 94 'ExportedSystemDirs': 'export_system_include_dirs', 95 'ExportedFlags': 'export_flags', 96 'Sanitize': 'sanitize', 97 'SanitizeMinimalDep': 'sanitize_minimal_dep', 98 'SanitizeUbsanDep': 'sanitize_ubsan_dep', 99 'Symlinks': 'symlinks', 100 'StaticExecutable': 'static_executable', 101 'InstallInRoot': 'install_in_root', 102 'InitRc': 'init_rc', 103 'VintfFragments': 'vintf_fragments', 104 'SharedLibs': 'shared_libs', 105 'StaticLibs': 'static_libs', 106 'RuntimeLibs': 'runtime_libs', 107 'Required': 'required', 108 'Filename': 'filename', 109 'CrateName': 'crate_name', 110 'Prebuilt': 'prebuilt', 111 'Overrides': 'overrides', 112 'MinSdkVersion': 'min_sdk_version', 113} 114 115LICENSE_KEYS = { 116 'LicenseKinds': 'license_kinds', 117 'LicenseTexts': 'license_text', 118} 119 120NOTICE_DIR = 'NOTICE_FILES' 121 122SANITIZER_VARIANT_PROPS = { 123 'export_include_dirs', 124 'export_system_include_dirs', 125 'export_flags', 126 'sanitize_minimal_dep', 127 'sanitize_ubsan_dep', 128 'src', 129} 130 131EXPORTED_FLAGS_PROPS = { 132 'export_include_dirs', 133 'export_system_include_dirs', 134 'export_flags', 135} 136 137# Convert json file to Android.bp prop dictionary 138def convert_json_to_bp_prop(json_path, bp_dir): 139 prop = json.load(json_path) 140 return convert_json_data_to_bp_prop(prop, bp_dir) 141 142# Converts parsed json dictionary (which is intermediate) to Android.bp prop 143# dictionary. This validates paths such as include directories and init_rc 144# files while converting. 145def convert_json_data_to_bp_prop(prop, bp_dir): 146 ret = {} 147 lic_ret = {} 148 149 module_name = prop['ModuleName'] 150 ret['name'] = module_name 151 152 # Soong will complain about non-existing paths on Android.bp. There might 153 # be missing files among generated header files, so check all exported 154 # directories and filter out invalid ones. Emits warning for such dirs. 155 # TODO: fix soong to track all generated headers correctly 156 for key in {'ExportedDirs', 'ExportedSystemDirs'}: 157 if key in prop: 158 prop[key] = remove_invalid_dirs(prop[key], bp_dir, module_name) 159 160 for key in prop: 161 if key in JSON_TO_BP: 162 ret[JSON_TO_BP[key]] = prop[key] 163 elif key in LICENSE_KEYS: 164 if key == 'LicenseTexts': 165 # Add path prefix 166 lic_ret[LICENSE_KEYS[key]] = [os.path.join(NOTICE_DIR, lic_path) 167 for lic_path in prop[key]] 168 else: 169 lic_ret[LICENSE_KEYS[key]] = prop[key] 170 else: 171 logging.warning('Unknown prop "%s" of module "%s"', key, 172 module_name) 173 174 return ret, lic_ret 175 176def is_64bit_arch(arch): 177 return '64' in arch # arm64, x86_64 178 179def remove_keys_from_dict(keys, d): 180 # May contain subdictionaries (e.g. cfi), so recursively erase 181 for k in list(d.keys()): 182 if k in keys: 183 del d[k] 184 elif type(d[k]) == dict: 185 remove_keys_from_dict(keys, d[k]) 186 187def reexport_vndk_header(name, arch_props): 188 remove_keys_from_dict(EXPORTED_FLAGS_PROPS, arch_props) 189 for arch in arch_props: 190 arch_props[arch]['shared_libs'] = [name] 191 arch_props[arch]['export_shared_lib_headers'] = [name] 192 193def gen_bp_module(image, variation, name, version, target_arch, vndk_list, arch_props, bp_dir): 194 # Generate Android.bp module for given snapshot. 195 # If a vndk library with the same name exists, reuses exported flags of the vndk library, 196 # instead of the snapshot's own flags. 197 198 if variation == 'license': 199 bp = 'license {\n' 200 bp += gen_bp_prop(arch_props, INDENT) 201 bp += '}\n\n' 202 return bp 203 204 prop = { 205 # These are common for all snapshot modules. 206 image: True, 207 'arch': {}, 208 } 209 210 if variation != 'etc': 211 prop['version'] = str(version) 212 prop['target_arch'] = target_arch 213 else: 214 prop['prefer'] = True 215 216 reexport_vndk_name = name 217 if reexport_vndk_name == "libc++_static": 218 reexport_vndk_name = "libc++" 219 220 if reexport_vndk_name in vndk_list: 221 if variation == 'shared': 222 logging.error("Module %s is both vendor snapshot shared and vndk" % name) 223 reexport_vndk_header(reexport_vndk_name, arch_props) 224 225 # Factor out common prop among architectures to minimize Android.bp. 226 common_prop = None 227 for arch in arch_props: 228 if common_prop is None: 229 common_prop = dict() 230 for k in arch_props[arch]: 231 common_prop[k] = arch_props[arch][k] 232 continue 233 for k in list(common_prop.keys()): 234 if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]: 235 del common_prop[k] 236 237 # Some keys has to be arch_props to prevent 32-bit only modules from being 238 # used as 64-bit modules, and vice versa. 239 for arch_prop_key in ['src', 'cfi']: 240 if arch_prop_key in common_prop: 241 del common_prop[arch_prop_key] 242 prop.update(common_prop) 243 244 has32 = has64 = False 245 stem32 = stem64 = '' 246 247 for arch in arch_props: 248 for k in common_prop: 249 if k in arch_props[arch]: 250 del arch_props[arch][k] 251 prop['arch'][arch] = arch_props[arch] 252 253 has64 |= is_64bit_arch(arch) 254 has32 |= not is_64bit_arch(arch) 255 256 # Record stem for snapshots. 257 # We don't check existence of 'src'; src must exist for executables 258 if variation == 'binary': 259 if is_64bit_arch(arch): 260 stem64 = os.path.basename(arch_props[arch]['src']) 261 else: 262 stem32 = os.path.basename(arch_props[arch]['src']) 263 264 # header snapshots doesn't need compile_multilib. The other snapshots, 265 # shared/static/object/binary snapshots, do need them 266 if variation != 'header' and variation != 'etc': 267 if has32 and has64: 268 prop['compile_multilib'] = 'both' 269 elif has32: 270 prop['compile_multilib'] = '32' 271 elif has64: 272 prop['compile_multilib'] = '64' 273 else: 274 raise RuntimeError("Module %s doesn't have prebuilts." % name) 275 276 # For binary snapshots, prefer 64bit if their stem collide and installing 277 # both is impossible 278 if variation == 'binary' and stem32 == stem64: 279 prop['compile_multilib'] = 'first' 280 281 module_type = '' 282 283 if variation == 'etc': 284 module_type = 'snapshot_etc' 285 else: 286 module_type = '%s_snapshot_%s' % (image, variation) 287 288 bp = module_type + ' {\n' 289 bp += gen_bp_prop(prop, INDENT) 290 bp += '}\n\n' 291 return bp 292 293def get_vndk_list(vndk_dir, target_arch): 294 """Generates vndk_libs list, e.g. ['libbase', 'libc++', ...] 295 This list is retrieved from vndk_dir/target_arch/configs/module_names.txt. 296 If it doesn't exist, print an error message and return an empty list. 297 """ 298 299 module_names_path = os.path.join(vndk_dir, target_arch, 'configs/module_names.txt') 300 301 try: 302 with open(module_names_path, 'r') as f: 303 """The format of module_names.txt is a list of "{so_name} {module_name}", e.g. 304 305 lib1.so lib1 306 lib2.so lib2 307 ... 308 309 We extract the module name part. 310 """ 311 return [l.split()[1] for l in f.read().strip('\n').split('\n')] 312 except IOError as e: 313 logging.error('Failed to read %s: %s' % (module_names_path, e.strerror)) 314 except IndexError as e: 315 logging.error('Failed to parse %s: invalid format' % module_names_path) 316 317 return [] 318 319def gen_bp_list_module(image, snapshot_version, vndk_list, target_arch, arch_props): 320 """Generates a {image}_snapshot module which contains lists of snapshots. 321 For vendor snapshot, vndk list is also included, extracted from vndk_dir. 322 """ 323 324 bp = '%s_snapshot {\n' % image 325 326 bp_props = dict() 327 bp_props['name'] = '%s_snapshot' % image 328 bp_props['version'] = str(snapshot_version) 329 if image == 'vendor': 330 bp_props['vndk_libs'] = vndk_list 331 332 variant_to_property = { 333 'shared': 'shared_libs', 334 'static': 'static_libs', 335 'rlib': 'rlibs', 336 'header': 'header_libs', 337 'binary': 'binaries', 338 'object': 'objects', 339 } 340 341 # arch_bp_prop[arch][variant_prop] = list 342 # e.g. arch_bp_prop['x86']['shared_libs'] == ['libfoo', 'libbar', ...] 343 arch_bp_prop = dict() 344 345 # Gather module lists per arch. 346 # arch_props structure: arch_props[variant][module_name][arch] 347 # e.g. arch_props['shared']['libc++']['x86'] 348 for variant in arch_props: 349 if variant in variant_to_property: 350 variant_name = variant_to_property[variant] 351 for name in arch_props[variant]: 352 for arch in arch_props[variant][name]: 353 if arch not in arch_bp_prop: 354 arch_bp_prop[arch] = dict() 355 if variant_name not in arch_bp_prop[arch]: 356 arch_bp_prop[arch][variant_name] = [] 357 arch_bp_prop[arch][variant_name].append(name) 358 359 bp_props['arch'] = arch_bp_prop 360 bp += gen_bp_prop(bp_props, INDENT) 361 362 bp += '}\n\n' 363 return bp 364 365def build_props(install_dir, image='', version=0 ): 366 # props[target_arch]["static"|"shared"|"binary"|"header"|"license"][name][arch] : json 367 props = dict() 368 369 # {target_arch}/{arch}/{variation}/{module}.json 370 for root, _, files in os.walk(install_dir, followlinks = True): 371 for file_name in sorted(files): 372 if not file_name.endswith('.json'): 373 continue 374 full_path = os.path.join(root, file_name) 375 rel_path = os.path.relpath(full_path, install_dir) 376 377 target_arch = get_target_arch(rel_path) 378 arch = get_arch(rel_path) 379 variation = get_variation(rel_path) 380 bp_dir = os.path.join(install_dir, target_arch) 381 382 if not target_arch in props: 383 props[target_arch] = dict() 384 props[target_arch]['license'] = dict() 385 if not variation in props[target_arch]: 386 props[target_arch][variation] = dict() 387 388 with open(full_path, 'r') as f: 389 prop, lic_prop = convert_json_to_bp_prop(f, bp_dir) 390 # Remove .json after parsing? 391 # os.unlink(full_path) 392 393 if variation != 'header': 394 prop['src'] = os.path.relpath( 395 rel_path[:-5], # removing .json 396 target_arch) 397 398 module_name = prop['name'] 399 400 # Is this sanitized variant? 401 if 'sanitize' in prop: 402 sanitizer_type = prop['sanitize'] 403 # module_name is {name}.{sanitizer_type}; trim sanitizer_type 404 module_name = module_name[:-len(sanitizer_type) - 1] 405 # Only leave props for the sanitize variant 406 for k in list(prop.keys()): 407 if not k in SANITIZER_VARIANT_PROPS: 408 del prop[k] 409 prop = {'name': module_name, sanitizer_type: prop} 410 411 if not lic_prop: 412 # This for the backward compatibility with the old snapshots 413 notice_path = os.path.join(NOTICE_DIR, module_name + '.txt') 414 if os.path.exists(os.path.join(bp_dir, notice_path)): 415 lic_prop['license_text'] = [notice_path] 416 417 # Update license props 418 if lic_prop and image and version: 419 lic_name = '{image}-v{version}-{name}-license'.format( 420 image=image, version=version, name=module_name) 421 if lic_name not in props[target_arch]['license']: 422 lic_prop['name'] = lic_name 423 props[target_arch]['license'][lic_name] = lic_prop 424 else: 425 props[target_arch]['license'][lic_name].update(lic_prop) 426 427 prop['licenses'] = [lic_name] 428 429 variation_dict = props[target_arch][variation] 430 if not module_name in variation_dict: 431 variation_dict[module_name] = dict() 432 if not arch in variation_dict[module_name]: 433 variation_dict[module_name][arch] = prop 434 else: 435 variation_dict[module_name][arch].update(prop) 436 437 return props 438 439def convert_json_host_data_to_bp(mod, install_dir, version): 440 """Create blueprint definition for a given host module. 441 442 All host modules are created as a cc_prebuilt_binary 443 blueprint module with the prefer attribute set to true. 444 445 Modules that already have a prebuilt are not created. 446 447 Args: 448 mod: JSON definition of the module 449 install_dir: installation directory of the host snapshot 450 version: the version of the host snapshot 451 """ 452 rust_proc_macro = mod.pop('RustProcMacro', False) 453 prop, lic_prop = convert_json_data_to_bp_prop(mod, install_dir) 454 if 'prebuilt' in prop: 455 return 456 457 if not rust_proc_macro: 458 prop['host_supported'] = True 459 prop['device_supported'] = False 460 prop['stl'] = 'none' 461 462 prop['prefer'] = True 463 ## Move install file to host source file 464 prop['target'] = dict() 465 prop['target']['host'] = dict() 466 prop['target']['host']['srcs'] = [prop['filename']] 467 del prop['filename'] 468 469 bp = '' 470 if lic_prop: 471 lic_name = 'host-v{version}-{name}-license'.format( 472 version=version, name=prop['name']) 473 lic_prop['name'] = lic_name 474 prop['licenses'] = [lic_name] 475 bp += 'license {\n' 476 bp += gen_bp_prop(lic_prop, INDENT) 477 bp += '}\n\n' 478 479 mod_type = 'cc_prebuilt_binary' 480 481 if rust_proc_macro: 482 mod_type = 'rust_prebuilt_proc_macro' 483 484 bp += mod_type + ' {\n' + gen_bp_prop(prop, INDENT) + '}\n\n' 485 return bp 486 487def gen_host_bp_file(install_dir, version): 488 """Generate Android.bp for a host snapshot. 489 490 This routine will find the JSON description file from a host 491 snapshot and create a blueprint definition for each module 492 and add to the created Android.bp file. 493 494 Args: 495 install_dir: directory where the host snapshot can be found 496 version: the version of the host snapshot 497 """ 498 bpfilename = 'Android.bp' 499 with open(os.path.join(install_dir, bpfilename), 'w') as wfp: 500 for file in os.listdir(install_dir): 501 if file.endswith('.json'): 502 with open(os.path.join(install_dir, file), 'r') as rfp: 503 props = json.load(rfp) 504 for mod in props: 505 prop = convert_json_host_data_to_bp(mod, install_dir, version) 506 if prop: 507 wfp.write(prop) 508 509def gen_bp_files(image, vndk_dir, install_dir, snapshot_version): 510 """Generates Android.bp for each archtecture. 511 Android.bp will contain a {image}_snapshot module having lists of VNDK and 512 vendor snapshot libraries, and {image}_snapshot_{variant} modules which are 513 prebuilt libraries of the snapshot. 514 515 Args: 516 image: string, name of partition (e.g. 'vendor', 'recovery') 517 vndk_dir: string, directory to which vndk snapshot is installed 518 install_dir: string, directory to which the snapshot will be installed 519 snapshot_version: int, version of the snapshot 520 """ 521 props = build_props(install_dir, image, snapshot_version) 522 523 for target_arch in sorted(props): 524 androidbp = '' 525 bp_dir = os.path.join(install_dir, target_arch) 526 vndk_list = [] 527 if image == 'vendor': 528 vndk_list = get_vndk_list(vndk_dir, target_arch) 529 530 # Generate snapshot modules. 531 for variation in sorted(props[target_arch]): 532 for name in sorted(props[target_arch][variation]): 533 androidbp += gen_bp_module(image, variation, name, 534 snapshot_version, target_arch, 535 vndk_list, 536 props[target_arch][variation][name], 537 bp_dir) 538 539 # Generate {image}_snapshot module which contains the list of modules. 540 androidbp += gen_bp_list_module(image, snapshot_version, vndk_list, 541 target_arch, props[target_arch]) 542 with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f: 543 logging.info('Generating Android.bp to: {}'.format(f.name)) 544 f.write(androidbp) 545 546 547def find_all_installed_files(install_dir): 548 installed_files = dict() 549 for root, _, files in os.walk(install_dir, followlinks = True): 550 for file_name in sorted(files): 551 if file_name.endswith('.json'): 552 continue 553 if file_name.endswith('Android.bp'): 554 continue 555 full_path = os.path.join(root, file_name) 556 size = os.stat(full_path).st_size 557 installed_files[full_path] = size 558 559 logging.debug('') 560 for f in sorted(installed_files.keys()): 561 logging.debug(f) 562 logging.debug('') 563 logging.debug('found {} installed files'.format(len(installed_files))) 564 logging.debug('') 565 return installed_files 566 567 568def find_files_in_props(target_arch, arch_install_dir, variation, name, props, file_to_info): 569 logging.debug('{} {} {} {} {}'.format( 570 target_arch, arch_install_dir, variation, name, props)) 571 572 def add_info(file, name, variation, arch, is_sanitized, is_header): 573 info = (name, variation, arch, is_sanitized, is_header) 574 info_list = file_to_info.get(file) 575 if not info_list: 576 info_list = [] 577 file_to_info[file] = info_list 578 info_list.append(info) 579 580 def find_file_in_list(dict, key, is_sanitized): 581 list = dict.get(key) 582 logging.debug(' {} {}'.format(key, list)) 583 if list: 584 for item in list: 585 item_path = os.path.join(arch_install_dir, item) 586 add_info(item_path, name, variation, arch, is_sanitized, False) 587 588 def find_file_in_dirs(dict, key, is_sanitized, is_header): 589 dirs = dict.get(key) 590 logging.debug(' {} {}'.format(key, dirs)) 591 if dirs: 592 for dir in dirs: 593 dir_path = os.path.join(arch_install_dir, dir) 594 logging.debug(' scanning {}'.format(dir_path)) 595 for root, _, files in os.walk(dir_path, followlinks = True): 596 for file_name in sorted(files): 597 item_path = os.path.join(root, file_name) 598 add_info(item_path, name, variation, arch, is_sanitized, is_header) 599 600 def find_file_in_dict(dict, is_sanitized): 601 logging.debug(' arch {}'.format(arch)) 602 logging.debug(' name {}'.format( name)) 603 logging.debug(' is_sanitized {}'.format(is_sanitized)) 604 605 src = dict.get('src') 606 logging.debug(' src {}'.format(src)) 607 if src: 608 src_path = os.path.join(arch_install_dir, src) 609 add_info(src_path, name, variation, arch, is_sanitized, False) 610 611 notice = dict.get('notice') 612 logging.debug(' notice {}'.format(notice)) 613 if notice: 614 notice_path = os.path.join(arch_install_dir, notice) 615 add_info(notice_path, name, variation, arch, is_sanitized, False) 616 617 find_file_in_list(dict, 'init_rc', is_sanitized) 618 find_file_in_list(dict, 'vintf_fragments', is_sanitized) 619 620 find_file_in_dirs(dict, 'export_include_dirs', is_sanitized, True) 621 find_file_in_dirs(dict, 'export_system_include_dirs', is_sanitized, True) 622 623 for arch in sorted(props): 624 name = props[arch]['name'] 625 find_file_in_dict(props[arch], False) 626 cfi = props[arch].get('cfi') 627 if cfi: 628 find_file_in_dict(cfi, True) 629 hwasan = props[arch].get('hwasan') 630 if hwasan: 631 find_file_in_dict(hwasan, True) 632 633 634def find_all_props_files(install_dir): 635 636 # This function builds a database of filename to module. This means that we 637 # need to dive into the json to find the files that the vendor snapshot 638 # provides, and link these back to modules that provide them. 639 640 file_to_info = dict() 641 642 props = build_props(install_dir) 643 for target_arch in sorted(props): 644 arch_install_dir = os.path.join(install_dir, target_arch) 645 for variation in sorted(props[target_arch]): 646 for name in sorted(props[target_arch][variation]): 647 find_files_in_props( 648 target_arch, 649 arch_install_dir, 650 variation, 651 name, 652 props[target_arch][variation][name], 653 file_to_info) 654 655 logging.debug('') 656 for f in sorted(file_to_info.keys()): 657 logging.debug(f) 658 logging.debug('') 659 logging.debug('found {} props files'.format(len(file_to_info))) 660 logging.debug('') 661 return file_to_info 662 663 664def get_ninja_inputs(ninja_binary, ninja_build_file, modules): 665 """Returns the set of input file path strings for the given modules. 666 667 Uses the `ninja -t inputs` tool. 668 669 Args: 670 ninja_binary: The path to a ninja binary. 671 ninja_build_file: The path to a .ninja file from a build. 672 modules: The list of modules to scan for inputs. 673 """ 674 inputs = set() 675 cmd = [ 676 ninja_binary, 677 "-f", 678 ninja_build_file, 679 "-t", 680 "inputs", 681 "-d", 682 ] + list(modules) 683 logging.debug('invoke ninja {}'.format(cmd)) 684 inputs = inputs.union(set( 685 subprocess.check_output(cmd).decode().strip("\n").split("\n"))) 686 687 return inputs 688 689def check_host_usage(install_dir, ninja_binary, ninja_file, goals, output): 690 """Find the host modules that are in the ninja build dependency for a given goal. 691 692 To use this routine the user has installed a fake host snapshot with all 693 possible host tools into the install directory. Check the ninja build 694 graph for any mapping to the modules in this installation directory. 695 696 This routine will print out a vsdk_host_tools = [] statement with the 697 dependent tools. The user can then use the vsdk_host_tools as the 698 deps of their host_snapshot module. 699 """ 700 file_to_info = dict() 701 for file in os.listdir(install_dir): 702 if file.endswith('.json'): 703 with open(os.path.join(install_dir,file),'r') as rfp: 704 props = json.load(rfp) 705 for mod in props: 706 file_to_info[os.path.join(install_dir,mod['Filename'])] = mod['ModuleName'] 707 708 used_modules = set() 709 ninja_inputs = get_ninja_inputs(ninja_binary, ninja_file, goals) 710 ## Check for host file in ninja inputs 711 for file in file_to_info: 712 if file in ninja_inputs: 713 used_modules.add(file_to_info[file]) 714 715 with open(output, 'w') as f: 716 f.write('vsdk_host_tools = [ \n') 717 for m in sorted(used_modules): 718 f.write(' "%s",\n' % m) 719 f.write('] \n') 720 721def check_module_usage(install_dir, ninja_binary, image, ninja_file, goals, 722 output): 723 all_installed_files = find_all_installed_files(install_dir) 724 all_props_files = find_all_props_files(install_dir) 725 726 ninja_inputs = get_ninja_inputs(ninja_binary, ninja_file, goals) 727 logging.debug('') 728 logging.debug('ninja inputs') 729 for ni in ninja_inputs: 730 logging.debug(ni) 731 732 logging.debug('found {} ninja_inputs for goals {}'.format( 733 len(ninja_inputs), goals)) 734 735 # Intersect the file_to_info dict with the ninja_inputs to determine 736 # which items from the vendor snapshot are actually used by the goals. 737 738 total_size = 0 739 used_size = 0 740 used_file_to_info = dict() 741 742 for file, size in all_installed_files.items(): 743 total_size += size 744 if file in ninja_inputs: 745 logging.debug('used: {}'.format(file)) 746 used_size += size 747 info = all_props_files.get(file) 748 749 if info: 750 used_file_to_info[file] = info 751 else: 752 logging.warning('No info for file {}'.format(file)) 753 used_file_to_info[file] = 'no info' 754 755 logging.debug('Total size {}'.format(total_size)) 756 logging.debug('Used size {}'.format(used_size)) 757 logging.debug('') 758 logging.debug('used items') 759 760 used_modules = set() 761 762 for f, i in sorted(used_file_to_info.items()): 763 logging.debug('{} {}'.format(f, i)) 764 for m in i: 765 (name, variation, arch, is_sanitized, is_header) = m 766 if not is_header: 767 used_modules.add(name) 768 769 with open(output, 'w') as f: 770 f.write('%s_SNAPSHOT_MODULES := \\\n' % image.upper()) 771 for m in sorted(used_modules): 772 f.write(' %s \\\n' % m) 773 774def check_call(cmd): 775 logging.debug('Running `{}`'.format(' '.join(cmd))) 776 subprocess.check_call(cmd) 777 778 779def fetch_artifact(branch, build, target, pattern, destination): 780 """Fetches build artifacts from Android Build server. 781 782 Args: 783 branch: string, branch to pull build artifacts from 784 build: string, build number to pull build artifacts from 785 target: string, target name to pull build artifacts from 786 pattern: string, pattern of build artifact file name 787 destination: string, destination to pull build artifact to 788 """ 789 fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' 790 cmd = [ 791 fetch_artifact_path, '--branch', branch, '--target', target, '--bid', 792 build, pattern, destination 793 ] 794 check_call(cmd) 795 796def install_artifacts(image, branch, build, target, local_dir, symlink, 797 install_dir): 798 """Installs vendor snapshot build artifacts to {install_dir}/v{version}. 799 800 1) Fetch build artifacts from Android Build server or from local_dir 801 2) Unzip or create symlinks to build artifacts 802 803 Args: 804 image: string, img file for which the snapshot was created (vendor, 805 recovery, etc.) 806 branch: string or None, branch name of build artifacts 807 build: string or None, build number of build artifacts 808 target: string or None, target name of build artifacts 809 local_dir: string or None, local dir to pull artifacts from 810 symlink: boolean, whether to use symlinks instead of unzipping the 811 vendor snapshot zip 812 install_dir: string, directory to install vendor snapshot 813 temp_artifact_dir: string, temp directory to hold build artifacts fetched 814 from Android Build server. For 'local' option, is set to None. 815 """ 816 artifact_pattern = image + '-*.zip' 817 818 def unzip_artifacts(artifact_dir): 819 artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) 820 for artifact in artifacts: 821 logging.info('Unzipping Vendor snapshot: {}'.format(artifact)) 822 check_call(['unzip', '-qn', artifact, '-d', install_dir]) 823 824 if branch and build and target: 825 with tempfile.TemporaryDirectory() as tmpdir: 826 logging.info( 827 'Fetching {pattern} from {branch} (bid: {build}, target: {target})' 828 .format( 829 pattern=artifact_pattern, 830 branch=branch, 831 build=build, 832 target=target)) 833 fetch_artifact(branch, build, target, artifact_pattern, tmpdir) 834 unzip_artifacts(tmpdir) 835 elif local_dir: 836 if symlink: 837 # This assumes local_dir is the location of vendor-snapshot in the 838 # build (e.g., out/soong/vendor-snapshot). 839 # 840 # Create the first level as proper directories and the next level 841 # as symlinks. 842 for item1 in os.listdir(local_dir): 843 dest_dir = os.path.join(install_dir, item1) 844 src_dir = os.path.join(local_dir, item1) 845 if os.path.isdir(src_dir): 846 check_call(['mkdir', '-p', dest_dir]) 847 # Create symlinks. 848 for item2 in os.listdir(src_dir): 849 src_item = os.path.join(src_dir, item2) 850 logging.info('Creating symlink from {} in {}'.format( 851 src_item, dest_dir)) 852 os.symlink(src_item, os.path.join(dest_dir, item2)) 853 else: 854 logging.info('Fetching local VNDK snapshot from {}'.format( 855 local_dir)) 856 unzip_artifacts(local_dir) 857 else: 858 raise RuntimeError('Neither local nor remote fetch information given.') 859 860def get_args(): 861 parser = argparse.ArgumentParser() 862 parser.add_argument( 863 'snapshot_version', 864 type=int, 865 help='Vendor snapshot version to install, e.g. "30".') 866 parser.add_argument( 867 '--image', 868 help=('Image whose snapshot is being updated (e.g., vendor, ' 869 'recovery , ramdisk, host, etc.)'), 870 default='vendor') 871 parser.add_argument('--branch', help='Branch to pull build from.') 872 parser.add_argument('--build', help='Build number to pull.') 873 parser.add_argument('--target', help='Target to pull.') 874 parser.add_argument( 875 '--local', 876 help=('Fetch local vendor snapshot artifacts from specified local ' 877 'directory instead of Android Build server. ' 878 'Example: --local /path/to/local/dir')) 879 parser.add_argument( 880 '--symlink', 881 action='store_true', 882 help='Use symlinks instead of unzipping vendor snapshot zip') 883 parser.add_argument( 884 '--install-dir', 885 required=True, 886 help=( 887 'Base directory to which vendor snapshot artifacts are installed. ' 888 'Example: --install-dir vendor/<company name>/vendor_snapshot/v30')) 889 parser.add_argument( 890 '--overwrite', 891 action='store_true', 892 help=( 893 'If provided, does not ask before overwriting the install-dir.')) 894 parser.add_argument( 895 '--check-module-usage', 896 action='store_true', 897 help='Check which modules are used.') 898 parser.add_argument( 899 '--check-module-usage-goal', 900 action='append', 901 help='Goal(s) for which --check-module-usage is calculated.') 902 parser.add_argument( 903 '--check-module-usage-ninja-file', 904 help='Ninja file for which --check-module-usage is calculated.') 905 parser.add_argument( 906 '--check-module-usage-output', 907 help='File to which to write the check-module-usage results.') 908 parser.add_argument( 909 '--vndk-dir', 910 help='Path to installed vndk snapshot directory. Needed to retrieve ' 911 'the list of VNDK. prebuilts/vndk/v{ver} will be used by default.') 912 913 parser.add_argument( 914 '-v', 915 '--verbose', 916 action='count', 917 default=0, 918 help='Increase output verbosity, e.g. "-v", "-vv".') 919 return parser.parse_args() 920 921 922def main(): 923 """Program entry point.""" 924 args = get_args() 925 926 host_image = args.image == 'host' 927 verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) 928 verbosity = min(args.verbose, 2) 929 logging.basicConfig( 930 format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', 931 level=verbose_map[verbosity]) 932 933 if not args.install_dir: 934 raise ValueError('Please provide --install-dir option.') 935 install_dir = os.path.expanduser(args.install_dir) 936 937 if args.check_module_usage: 938 ninja_binary = './prebuilts/build-tools/linux-x86/bin/ninja' 939 940 if not args.check_module_usage_goal: 941 raise ValueError('Please provide --check-module-usage-goal option.') 942 if not args.check_module_usage_ninja_file: 943 raise ValueError( 944 'Please provide --check-module-usage-ninja-file option.') 945 if not args.check_module_usage_output: 946 raise ValueError( 947 'Please provide --check-module-usage-output option.') 948 949 if host_image: 950 check_host_usage(install_dir, ninja_binary, 951 args.check_module_usage_ninja_file, 952 args.check_module_usage_goal, 953 args.check_module_usage_output) 954 else: 955 check_module_usage(install_dir, ninja_binary, args.image, 956 args.check_module_usage_ninja_file, 957 args.check_module_usage_goal, 958 args.check_module_usage_output) 959 return 960 961 local = None 962 if args.local: 963 local = os.path.expanduser(args.local) 964 965 if local: 966 if args.build or args.branch or args.target: 967 raise ValueError( 968 'When --local option is set, --branch, --build or --target cannot be ' 969 'specified.') 970 elif not os.path.isdir(local): 971 raise RuntimeError( 972 'The specified local directory, {}, does not exist.'.format( 973 local)) 974 else: 975 if not (args.build and args.branch and args.target): 976 raise ValueError( 977 'Please provide --branch, --build and --target. Or set --local ' 978 'option.') 979 980 snapshot_version = args.snapshot_version 981 982 if os.path.exists(install_dir): 983 def remove_dir(): 984 logging.info('Removing {}'.format(install_dir)) 985 check_call(['rm', '-rf', install_dir]) 986 if args.overwrite: 987 remove_dir() 988 else: 989 resp = input('Directory {} already exists. IT WILL BE REMOVED.\n' 990 'Are you sure? (yes/no): '.format(install_dir)) 991 if resp == 'yes': 992 remove_dir() 993 elif resp == 'no': 994 logging.info('Cancelled snapshot install.') 995 return 996 else: 997 raise ValueError('Did not understand: ' + resp) 998 check_call(['mkdir', '-p', install_dir]) 999 1000 if args.vndk_dir: 1001 vndk_dir = os.path.expanduser(args.vndk_dir) 1002 else: 1003 vndk_dir = 'prebuilts/vndk/v%d' % snapshot_version 1004 logging.debug('Using %s for vndk directory' % vndk_dir) 1005 1006 install_artifacts( 1007 image=args.image, 1008 branch=args.branch, 1009 build=args.build, 1010 target=args.target, 1011 local_dir=local, 1012 symlink=args.symlink, 1013 install_dir=install_dir) 1014 1015 if host_image: 1016 gen_host_bp_file(install_dir, snapshot_version) 1017 1018 else: 1019 gen_bp_files(args.image, vndk_dir, install_dir, snapshot_version) 1020 1021if __name__ == '__main__': 1022 main() 1023