• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Unzips and installs the vendor snapshot."""
18
19import argparse
20import glob
21import logging
22import os
23import re
24import shutil
25import subprocess
26import sys
27import tempfile
28import textwrap
29import json
30
31INDENT = ' ' * 4
32
33
34def get_target_arch(json_rel_path):
35    return json_rel_path.split('/')[0]
36
37
38def get_arch(json_rel_path):
39    return json_rel_path.split('/')[1].split('-')[1]
40
41
42def get_variation(json_rel_path):
43    return json_rel_path.split('/')[2]
44
45# convert .bp prop dictionary to .bp prop string
46def gen_bp_prop(prop, ind):
47    bp = ''
48    for key in sorted(prop):
49        val = prop[key]
50
51        # Skip empty list or dict, rather than printing empty prop like
52        # "key: []," or "key: {},"
53        if type(val) == list and len(val) == 0:
54            continue
55        if type(val) == dict and gen_bp_prop(val, '') == '':
56            continue
57
58        bp += ind + key + ': '
59        if type(val) == bool:
60            bp += 'true,\n' if val else 'false,\n'
61        elif type(val) == str:
62            bp += '"%s",\n' % val
63        elif type(val) == list:
64            bp += '[\n'
65            for elem in val:
66                bp += ind + INDENT + '"%s",\n' % elem
67            bp += ind + '],\n'
68        elif type(val) == dict:
69            bp += '{\n'
70            bp += gen_bp_prop(val, ind + INDENT)
71            bp += ind + '},\n'
72        else:
73            raise TypeError('unsupported type %s for gen_bp_prop' % type(val))
74    return bp
75
76
77# Remove non-existent dirs from given list. Emits warning for such dirs.
78def remove_invalid_dirs(paths, bp_dir, module_name):
79    ret = []
80    for path in paths:
81        if os.path.isdir(os.path.join(bp_dir, path)):
82            ret.append(path)
83        else:
84            logging.warning('Dir "%s" of module "%s" does not exist', path,
85                            module_name)
86    return ret
87
88
89JSON_TO_BP = {
90    'ModuleName': 'name',
91    'ModuleStemName': 'stem',
92    'RelativeInstallPath': 'relative_install_path',
93    'ExportedDirs': 'export_include_dirs',
94    'ExportedSystemDirs': 'export_system_include_dirs',
95    'ExportedFlags': 'export_flags',
96    'Sanitize': 'sanitize',
97    'SanitizeMinimalDep': 'sanitize_minimal_dep',
98    'SanitizeUbsanDep': 'sanitize_ubsan_dep',
99    'Symlinks': 'symlinks',
100    'StaticExecutable': 'static_executable',
101    'InstallInRoot': 'install_in_root',
102    'InitRc': 'init_rc',
103    'VintfFragments': 'vintf_fragments',
104    'SharedLibs': 'shared_libs',
105    'StaticLibs': 'static_libs',
106    'RuntimeLibs': 'runtime_libs',
107    'Required': 'required',
108    'Filename': 'filename',
109    'CrateName': 'crate_name',
110    'Prebuilt': 'prebuilt',
111    'Overrides': 'overrides',
112    'MinSdkVersion': 'min_sdk_version',
113}
114
115LICENSE_KEYS = {
116    'LicenseKinds': 'license_kinds',
117    'LicenseTexts': 'license_text',
118}
119
120NOTICE_DIR = 'NOTICE_FILES'
121
122SANITIZER_VARIANT_PROPS = {
123    'export_include_dirs',
124    'export_system_include_dirs',
125    'export_flags',
126    'sanitize_minimal_dep',
127    'sanitize_ubsan_dep',
128    'src',
129}
130
131EXPORTED_FLAGS_PROPS = {
132    'export_include_dirs',
133    'export_system_include_dirs',
134    'export_flags',
135}
136
137# Convert json file to Android.bp prop dictionary
138def convert_json_to_bp_prop(json_path, bp_dir):
139    prop = json.load(json_path)
140    return convert_json_data_to_bp_prop(prop, bp_dir)
141
142# Converts parsed json dictionary (which is intermediate) to Android.bp prop
143# dictionary. This validates paths such as include directories and init_rc
144# files while converting.
145def convert_json_data_to_bp_prop(prop, bp_dir):
146    ret = {}
147    lic_ret = {}
148
149    module_name = prop['ModuleName']
150    ret['name'] = module_name
151
152    # Soong will complain about non-existing paths on Android.bp. There might
153    # be missing files among generated header files, so check all exported
154    # directories and filter out invalid ones. Emits warning for such dirs.
155    # TODO: fix soong to track all generated headers correctly
156    for key in {'ExportedDirs', 'ExportedSystemDirs'}:
157        if key in prop:
158            prop[key] = remove_invalid_dirs(prop[key], bp_dir, module_name)
159
160    for key in prop:
161        if key in JSON_TO_BP:
162            ret[JSON_TO_BP[key]] = prop[key]
163        elif key in LICENSE_KEYS:
164            if key == 'LicenseTexts':
165                # Add path prefix
166                lic_ret[LICENSE_KEYS[key]] = [os.path.join(NOTICE_DIR, lic_path)
167                                              for lic_path in prop[key]]
168            else:
169                lic_ret[LICENSE_KEYS[key]] = prop[key]
170        else:
171            logging.warning('Unknown prop "%s" of module "%s"', key,
172                            module_name)
173
174    return ret, lic_ret
175
176def is_64bit_arch(arch):
177    return '64' in arch # arm64, x86_64
178
179def remove_keys_from_dict(keys, d):
180    # May contain subdictionaries (e.g. cfi), so recursively erase
181    for k in list(d.keys()):
182        if k in keys:
183            del d[k]
184        elif type(d[k]) == dict:
185            remove_keys_from_dict(keys, d[k])
186
187def reexport_vndk_header(name, arch_props):
188    remove_keys_from_dict(EXPORTED_FLAGS_PROPS, arch_props)
189    for arch in arch_props:
190        arch_props[arch]['shared_libs'] = [name]
191        arch_props[arch]['export_shared_lib_headers'] = [name]
192
193def gen_bp_module(image, variation, name, version, target_arch, vndk_list, arch_props, bp_dir):
194    # Generate Android.bp module for given snapshot.
195    # If a vndk library with the same name exists, reuses exported flags of the vndk library,
196    # instead of the snapshot's own flags.
197
198    if variation == 'license':
199        bp = 'license {\n'
200        bp += gen_bp_prop(arch_props, INDENT)
201        bp += '}\n\n'
202        return bp
203
204    prop = {
205        # These are common for all snapshot modules.
206        image: True,
207        'arch': {},
208    }
209
210    if variation != 'etc':
211        prop['version'] = str(version)
212        prop['target_arch'] = target_arch
213    else:
214        prop['prefer'] = True
215
216    reexport_vndk_name = name
217    if reexport_vndk_name == "libc++_static":
218        reexport_vndk_name = "libc++"
219
220    if reexport_vndk_name in vndk_list:
221        if variation == 'shared':
222            logging.error("Module %s is both vendor snapshot shared and vndk" % name)
223        reexport_vndk_header(reexport_vndk_name, arch_props)
224
225    # Factor out common prop among architectures to minimize Android.bp.
226    common_prop = None
227    for arch in arch_props:
228        if common_prop is None:
229            common_prop = dict()
230            for k in arch_props[arch]:
231                common_prop[k] = arch_props[arch][k]
232            continue
233        for k in list(common_prop.keys()):
234            if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
235                del common_prop[k]
236
237    # Some keys has to be arch_props to prevent 32-bit only modules from being
238    # used as 64-bit modules, and vice versa.
239    for arch_prop_key in ['src', 'cfi']:
240        if arch_prop_key in common_prop:
241            del common_prop[arch_prop_key]
242    prop.update(common_prop)
243
244    has32 = has64 = False
245    stem32 = stem64 = ''
246
247    for arch in arch_props:
248        for k in common_prop:
249            if k in arch_props[arch]:
250                del arch_props[arch][k]
251        prop['arch'][arch] = arch_props[arch]
252
253        has64 |= is_64bit_arch(arch)
254        has32 |= not is_64bit_arch(arch)
255
256        # Record stem for snapshots.
257        # We don't check existence of 'src'; src must exist for executables
258        if variation == 'binary':
259            if is_64bit_arch(arch):
260                stem64 = os.path.basename(arch_props[arch]['src'])
261            else:
262                stem32 = os.path.basename(arch_props[arch]['src'])
263
264    # header snapshots doesn't need compile_multilib. The other snapshots,
265    # shared/static/object/binary snapshots, do need them
266    if variation != 'header' and variation != 'etc':
267        if has32 and has64:
268            prop['compile_multilib'] = 'both'
269        elif has32:
270            prop['compile_multilib'] = '32'
271        elif has64:
272            prop['compile_multilib'] = '64'
273        else:
274            raise RuntimeError("Module %s doesn't have prebuilts." % name)
275
276    # For binary snapshots, prefer 64bit if their stem collide and installing
277    # both is impossible
278    if variation == 'binary' and stem32 == stem64:
279        prop['compile_multilib'] = 'first'
280
281    module_type = ''
282
283    if variation == 'etc':
284        module_type = 'snapshot_etc'
285    else:
286        module_type = '%s_snapshot_%s' % (image, variation)
287
288    bp = module_type + ' {\n'
289    bp += gen_bp_prop(prop, INDENT)
290    bp += '}\n\n'
291    return bp
292
293def get_vndk_list(vndk_dir, target_arch):
294    """Generates vndk_libs list, e.g. ['libbase', 'libc++', ...]
295    This list is retrieved from vndk_dir/target_arch/configs/module_names.txt.
296    If it doesn't exist, print an error message and return an empty list.
297    """
298
299    module_names_path = os.path.join(vndk_dir, target_arch, 'configs/module_names.txt')
300
301    try:
302        with open(module_names_path, 'r') as f:
303            """The format of module_names.txt is a list of "{so_name} {module_name}", e.g.
304
305                lib1.so lib1
306                lib2.so lib2
307                ...
308
309            We extract the module name part.
310            """
311            return [l.split()[1] for l in f.read().strip('\n').split('\n')]
312    except IOError as e:
313        logging.error('Failed to read %s: %s' % (module_names_path, e.strerror))
314    except IndexError as e:
315        logging.error('Failed to parse %s: invalid format' % module_names_path)
316
317    return []
318
319def gen_bp_list_module(image, snapshot_version, vndk_list, target_arch, arch_props):
320    """Generates a {image}_snapshot module which contains lists of snapshots.
321    For vendor snapshot, vndk list is also included, extracted from vndk_dir.
322    """
323
324    bp = '%s_snapshot {\n' % image
325
326    bp_props = dict()
327    bp_props['name'] = '%s_snapshot' % image
328    bp_props['version'] = str(snapshot_version)
329    if image == 'vendor':
330        bp_props['vndk_libs'] = vndk_list
331
332    variant_to_property = {
333        'shared': 'shared_libs',
334        'static': 'static_libs',
335        'rlib': 'rlibs',
336        'header': 'header_libs',
337        'binary': 'binaries',
338        'object': 'objects',
339    }
340
341    # arch_bp_prop[arch][variant_prop] = list
342    # e.g. arch_bp_prop['x86']['shared_libs'] == ['libfoo', 'libbar', ...]
343    arch_bp_prop = dict()
344
345    # Gather module lists per arch.
346    # arch_props structure: arch_props[variant][module_name][arch]
347    # e.g. arch_props['shared']['libc++']['x86']
348    for variant in arch_props:
349        if variant in variant_to_property:
350            variant_name = variant_to_property[variant]
351            for name in arch_props[variant]:
352                for arch in arch_props[variant][name]:
353                    if arch not in arch_bp_prop:
354                        arch_bp_prop[arch] = dict()
355                    if variant_name not in arch_bp_prop[arch]:
356                        arch_bp_prop[arch][variant_name] = []
357                    arch_bp_prop[arch][variant_name].append(name)
358
359    bp_props['arch'] = arch_bp_prop
360    bp += gen_bp_prop(bp_props, INDENT)
361
362    bp += '}\n\n'
363    return bp
364
365def build_props(install_dir, image='', version=0 ):
366    # props[target_arch]["static"|"shared"|"binary"|"header"|"license"][name][arch] : json
367    props = dict()
368
369    # {target_arch}/{arch}/{variation}/{module}.json
370    for root, _, files in os.walk(install_dir, followlinks = True):
371        for file_name in sorted(files):
372            if not file_name.endswith('.json'):
373                continue
374            full_path = os.path.join(root, file_name)
375            rel_path = os.path.relpath(full_path, install_dir)
376
377            target_arch = get_target_arch(rel_path)
378            arch = get_arch(rel_path)
379            variation = get_variation(rel_path)
380            bp_dir = os.path.join(install_dir, target_arch)
381
382            if not target_arch in props:
383                props[target_arch] = dict()
384                props[target_arch]['license'] = dict()
385            if not variation in props[target_arch]:
386                props[target_arch][variation] = dict()
387
388            with open(full_path, 'r') as f:
389                prop, lic_prop = convert_json_to_bp_prop(f, bp_dir)
390                # Remove .json after parsing?
391                # os.unlink(full_path)
392
393            if variation != 'header':
394                prop['src'] = os.path.relpath(
395                    rel_path[:-5],  # removing .json
396                    target_arch)
397
398            module_name = prop['name']
399
400            # Is this sanitized variant?
401            if 'sanitize' in prop:
402                sanitizer_type = prop['sanitize']
403                # module_name is {name}.{sanitizer_type}; trim sanitizer_type
404                module_name = module_name[:-len(sanitizer_type) - 1]
405                # Only leave props for the sanitize variant
406                for k in list(prop.keys()):
407                    if not k in SANITIZER_VARIANT_PROPS:
408                        del prop[k]
409                prop = {'name': module_name, sanitizer_type: prop}
410
411            if not lic_prop:
412                # This for the backward compatibility with the old snapshots
413                notice_path = os.path.join(NOTICE_DIR, module_name + '.txt')
414                if os.path.exists(os.path.join(bp_dir, notice_path)):
415                    lic_prop['license_text'] = [notice_path]
416
417            # Update license props
418            if lic_prop and image and version:
419                lic_name = '{image}-v{version}-{name}-license'.format(
420                    image=image, version=version, name=module_name)
421                if lic_name not in props[target_arch]['license']:
422                    lic_prop['name'] = lic_name
423                    props[target_arch]['license'][lic_name] = lic_prop
424                else:
425                    props[target_arch]['license'][lic_name].update(lic_prop)
426
427                prop['licenses'] = [lic_name]
428
429            variation_dict = props[target_arch][variation]
430            if not module_name in variation_dict:
431                variation_dict[module_name] = dict()
432            if not arch in variation_dict[module_name]:
433                variation_dict[module_name][arch] = prop
434            else:
435                variation_dict[module_name][arch].update(prop)
436
437    return props
438
439def convert_json_host_data_to_bp(mod, install_dir, version):
440    """Create blueprint definition for a given host module.
441
442    All host modules are created as a cc_prebuilt_binary
443    blueprint module with the prefer attribute set to true.
444
445    Modules that already have a prebuilt are not created.
446
447    Args:
448      mod: JSON definition of the module
449      install_dir: installation directory of the host snapshot
450      version: the version of the host snapshot
451    """
452    rust_proc_macro = mod.pop('RustProcMacro', False)
453    prop, lic_prop = convert_json_data_to_bp_prop(mod, install_dir)
454    if 'prebuilt' in prop:
455        return
456
457    if not rust_proc_macro:
458        prop['host_supported'] = True
459        prop['device_supported'] = False
460        prop['stl'] = 'none'
461
462    prop['prefer'] = True
463    ## Move install file to host source file
464    prop['target'] = dict()
465    prop['target']['host'] = dict()
466    prop['target']['host']['srcs'] = [prop['filename']]
467    del prop['filename']
468
469    bp = ''
470    if lic_prop:
471        lic_name = 'host-v{version}-{name}-license'.format(
472                    version=version, name=prop['name'])
473        lic_prop['name'] = lic_name
474        prop['licenses'] = [lic_name]
475        bp += 'license {\n'
476        bp += gen_bp_prop(lic_prop, INDENT)
477        bp += '}\n\n'
478
479    mod_type = 'cc_prebuilt_binary'
480
481    if rust_proc_macro:
482        mod_type = 'rust_prebuilt_proc_macro'
483
484    bp += mod_type + ' {\n' + gen_bp_prop(prop, INDENT) + '}\n\n'
485    return bp
486
487def gen_host_bp_file(install_dir, version):
488    """Generate Android.bp for a host snapshot.
489
490    This routine will find the JSON description file from a host
491    snapshot and create a blueprint definition for each module
492    and add to the created Android.bp file.
493
494    Args:
495      install_dir: directory where the host snapshot can be found
496      version: the version of the host snapshot
497    """
498    bpfilename = 'Android.bp'
499    with open(os.path.join(install_dir, bpfilename), 'w') as wfp:
500        for file in os.listdir(install_dir):
501            if file.endswith('.json'):
502                with open(os.path.join(install_dir, file), 'r') as rfp:
503                    props = json.load(rfp)
504                    for mod in props:
505                        prop = convert_json_host_data_to_bp(mod, install_dir, version)
506                        if prop:
507                            wfp.write(prop)
508
509def gen_bp_files(image, vndk_dir, install_dir, snapshot_version):
510    """Generates Android.bp for each archtecture.
511    Android.bp will contain a {image}_snapshot module having lists of VNDK and
512    vendor snapshot libraries, and {image}_snapshot_{variant} modules which are
513    prebuilt libraries of the snapshot.
514
515    Args:
516      image: string, name of partition (e.g. 'vendor', 'recovery')
517      vndk_dir: string, directory to which vndk snapshot is installed
518      install_dir: string, directory to which the snapshot will be installed
519      snapshot_version: int, version of the snapshot
520    """
521    props = build_props(install_dir, image, snapshot_version)
522
523    for target_arch in sorted(props):
524        androidbp = ''
525        bp_dir = os.path.join(install_dir, target_arch)
526        vndk_list = []
527        if image == 'vendor':
528            vndk_list = get_vndk_list(vndk_dir, target_arch)
529
530        # Generate snapshot modules.
531        for variation in sorted(props[target_arch]):
532            for name in sorted(props[target_arch][variation]):
533                androidbp += gen_bp_module(image, variation, name,
534                                           snapshot_version, target_arch,
535                                           vndk_list,
536                                           props[target_arch][variation][name],
537                                           bp_dir)
538
539        # Generate {image}_snapshot module which contains the list of modules.
540        androidbp += gen_bp_list_module(image, snapshot_version, vndk_list,
541                                        target_arch, props[target_arch])
542        with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f:
543            logging.info('Generating Android.bp to: {}'.format(f.name))
544            f.write(androidbp)
545
546
547def find_all_installed_files(install_dir):
548    installed_files = dict()
549    for root, _, files in os.walk(install_dir, followlinks = True):
550        for file_name in sorted(files):
551            if file_name.endswith('.json'):
552                continue
553            if file_name.endswith('Android.bp'):
554                continue
555            full_path = os.path.join(root, file_name)
556            size = os.stat(full_path).st_size
557            installed_files[full_path] = size
558
559    logging.debug('')
560    for f in sorted(installed_files.keys()):
561        logging.debug(f)
562    logging.debug('')
563    logging.debug('found {} installed files'.format(len(installed_files)))
564    logging.debug('')
565    return installed_files
566
567
568def find_files_in_props(target_arch, arch_install_dir, variation, name, props, file_to_info):
569    logging.debug('{} {} {} {} {}'.format(
570        target_arch, arch_install_dir, variation, name, props))
571
572    def add_info(file, name, variation, arch, is_sanitized, is_header):
573        info = (name, variation, arch, is_sanitized, is_header)
574        info_list = file_to_info.get(file)
575        if not info_list:
576            info_list = []
577            file_to_info[file] = info_list
578        info_list.append(info)
579
580    def find_file_in_list(dict, key, is_sanitized):
581        list = dict.get(key)
582        logging.debug('    {} {}'.format(key, list))
583        if list:
584            for item in list:
585                item_path = os.path.join(arch_install_dir, item)
586                add_info(item_path, name, variation, arch, is_sanitized, False)
587
588    def find_file_in_dirs(dict, key, is_sanitized, is_header):
589        dirs = dict.get(key)
590        logging.debug('    {} {}'.format(key, dirs))
591        if dirs:
592            for dir in dirs:
593                dir_path = os.path.join(arch_install_dir, dir)
594                logging.debug('        scanning {}'.format(dir_path))
595                for root, _, files in os.walk(dir_path, followlinks = True):
596                    for file_name in sorted(files):
597                        item_path = os.path.join(root, file_name)
598                        add_info(item_path, name, variation, arch, is_sanitized, is_header)
599
600    def find_file_in_dict(dict, is_sanitized):
601        logging.debug('    arch {}'.format(arch))
602        logging.debug('    name {}'.format( name))
603        logging.debug('    is_sanitized {}'.format(is_sanitized))
604
605        src = dict.get('src')
606        logging.debug('    src {}'.format(src))
607        if src:
608            src_path = os.path.join(arch_install_dir, src)
609            add_info(src_path, name, variation, arch, is_sanitized, False)
610
611        notice = dict.get('notice')
612        logging.debug('    notice {}'.format(notice))
613        if notice:
614            notice_path = os.path.join(arch_install_dir, notice)
615            add_info(notice_path, name, variation, arch, is_sanitized, False)
616
617        find_file_in_list(dict, 'init_rc', is_sanitized)
618        find_file_in_list(dict, 'vintf_fragments', is_sanitized)
619
620        find_file_in_dirs(dict, 'export_include_dirs', is_sanitized, True)
621        find_file_in_dirs(dict, 'export_system_include_dirs', is_sanitized, True)
622
623    for arch in sorted(props):
624        name = props[arch]['name']
625        find_file_in_dict(props[arch], False)
626        cfi = props[arch].get('cfi')
627        if cfi:
628            find_file_in_dict(cfi, True)
629        hwasan = props[arch].get('hwasan')
630        if hwasan:
631            find_file_in_dict(hwasan, True)
632
633
634def find_all_props_files(install_dir):
635
636    # This function builds a database of filename to module. This means that we
637    # need to dive into the json to find the files that the vendor snapshot
638    # provides, and link these back to modules that provide them.
639
640    file_to_info = dict()
641
642    props = build_props(install_dir)
643    for target_arch in sorted(props):
644        arch_install_dir = os.path.join(install_dir, target_arch)
645        for variation in sorted(props[target_arch]):
646            for name in sorted(props[target_arch][variation]):
647                find_files_in_props(
648                    target_arch,
649                    arch_install_dir,
650                    variation,
651                    name,
652                    props[target_arch][variation][name],
653                    file_to_info)
654
655    logging.debug('')
656    for f in sorted(file_to_info.keys()):
657        logging.debug(f)
658    logging.debug('')
659    logging.debug('found {} props files'.format(len(file_to_info)))
660    logging.debug('')
661    return file_to_info
662
663
664def get_ninja_inputs(ninja_binary, ninja_build_file, modules):
665    """Returns the set of input file path strings for the given modules.
666
667    Uses the `ninja -t inputs` tool.
668
669    Args:
670        ninja_binary: The path to a ninja binary.
671        ninja_build_file: The path to a .ninja file from a build.
672        modules: The list of modules to scan for inputs.
673    """
674    inputs = set()
675    cmd = [
676        ninja_binary,
677        "-f",
678        ninja_build_file,
679        "-t",
680        "inputs",
681        "-d",
682    ] + list(modules)
683    logging.debug('invoke ninja {}'.format(cmd))
684    inputs = inputs.union(set(
685        subprocess.check_output(cmd).decode().strip("\n").split("\n")))
686
687    return inputs
688
689def check_host_usage(install_dir, ninja_binary, ninja_file, goals, output):
690    """Find the host modules that are in the ninja build dependency for a given goal.
691
692    To use this routine the user has installed a fake host snapshot with all
693    possible host tools into the install directory.  Check the ninja build
694    graph for any mapping to the modules in this installation directory.
695
696    This routine will print out a vsdk_host_tools = [] statement with the
697    dependent tools.  The user can then use the vsdk_host_tools as the
698    deps of their host_snapshot module.
699    """
700    file_to_info = dict()
701    for file in os.listdir(install_dir):
702        if file.endswith('.json'):
703            with open(os.path.join(install_dir,file),'r') as rfp:
704                props = json.load(rfp)
705                for mod in props:
706                    file_to_info[os.path.join(install_dir,mod['Filename'])] = mod['ModuleName']
707
708    used_modules = set()
709    ninja_inputs = get_ninja_inputs(ninja_binary, ninja_file, goals)
710    ## Check for host file in ninja inputs
711    for file in file_to_info:
712        if file in ninja_inputs:
713            used_modules.add(file_to_info[file])
714
715    with open(output, 'w') as f:
716        f.write('vsdk_host_tools = [ \n')
717        for m in sorted(used_modules):
718            f.write('  "%s",\n' % m)
719        f.write('] \n')
720
721def check_module_usage(install_dir, ninja_binary, image, ninja_file, goals,
722                       output):
723    all_installed_files = find_all_installed_files(install_dir)
724    all_props_files = find_all_props_files(install_dir)
725
726    ninja_inputs = get_ninja_inputs(ninja_binary, ninja_file, goals)
727    logging.debug('')
728    logging.debug('ninja inputs')
729    for ni in ninja_inputs:
730        logging.debug(ni)
731
732    logging.debug('found {} ninja_inputs for goals {}'.format(
733        len(ninja_inputs), goals))
734
735    # Intersect the file_to_info dict with the ninja_inputs to determine
736    # which items from the vendor snapshot are actually used by the goals.
737
738    total_size = 0
739    used_size = 0
740    used_file_to_info = dict()
741
742    for file, size in all_installed_files.items():
743        total_size += size
744        if file in ninja_inputs:
745            logging.debug('used: {}'.format(file))
746            used_size += size
747            info = all_props_files.get(file)
748
749            if info:
750                used_file_to_info[file] = info
751            else:
752                logging.warning('No info for file {}'.format(file))
753                used_file_to_info[file] = 'no info'
754
755    logging.debug('Total size {}'.format(total_size))
756    logging.debug('Used size {}'.format(used_size))
757    logging.debug('')
758    logging.debug('used items')
759
760    used_modules = set()
761
762    for f, i in sorted(used_file_to_info.items()):
763        logging.debug('{} {}'.format(f, i))
764        for m in i:
765            (name, variation, arch, is_sanitized, is_header) = m
766            if not is_header:
767                used_modules.add(name)
768
769    with open(output, 'w') as f:
770        f.write('%s_SNAPSHOT_MODULES := \\\n' % image.upper())
771        for m in sorted(used_modules):
772            f.write('  %s \\\n' % m)
773
774def check_call(cmd):
775    logging.debug('Running `{}`'.format(' '.join(cmd)))
776    subprocess.check_call(cmd)
777
778
779def fetch_artifact(branch, build, target, pattern, destination):
780    """Fetches build artifacts from Android Build server.
781
782    Args:
783      branch: string, branch to pull build artifacts from
784      build: string, build number to pull build artifacts from
785      target: string, target name to pull build artifacts from
786      pattern: string, pattern of build artifact file name
787      destination: string, destination to pull build artifact to
788    """
789    fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
790    cmd = [
791        fetch_artifact_path, '--branch', branch, '--target', target, '--bid',
792        build, pattern, destination
793    ]
794    check_call(cmd)
795
796def install_artifacts(image, branch, build, target, local_dir, symlink,
797                      install_dir):
798    """Installs vendor snapshot build artifacts to {install_dir}/v{version}.
799
800    1) Fetch build artifacts from Android Build server or from local_dir
801    2) Unzip or create symlinks to build artifacts
802
803    Args:
804      image: string, img file for which the snapshot was created (vendor,
805             recovery, etc.)
806      branch: string or None, branch name of build artifacts
807      build: string or None, build number of build artifacts
808      target: string or None, target name of build artifacts
809      local_dir: string or None, local dir to pull artifacts from
810      symlink: boolean, whether to use symlinks instead of unzipping the
811        vendor snapshot zip
812      install_dir: string, directory to install vendor snapshot
813      temp_artifact_dir: string, temp directory to hold build artifacts fetched
814        from Android Build server. For 'local' option, is set to None.
815    """
816    artifact_pattern = image + '-*.zip'
817
818    def unzip_artifacts(artifact_dir):
819        artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
820        for artifact in artifacts:
821            logging.info('Unzipping Vendor snapshot: {}'.format(artifact))
822            check_call(['unzip', '-qn', artifact, '-d', install_dir])
823
824    if branch and build and target:
825        with tempfile.TemporaryDirectory() as tmpdir:
826            logging.info(
827                'Fetching {pattern} from {branch} (bid: {build}, target: {target})'
828                .format(
829                    pattern=artifact_pattern,
830                    branch=branch,
831                    build=build,
832                    target=target))
833            fetch_artifact(branch, build, target, artifact_pattern, tmpdir)
834            unzip_artifacts(tmpdir)
835    elif local_dir:
836        if symlink:
837            # This assumes local_dir is the location of vendor-snapshot in the
838            # build (e.g., out/soong/vendor-snapshot).
839            #
840            # Create the first level as proper directories and the next level
841            # as symlinks.
842            for item1 in os.listdir(local_dir):
843                dest_dir = os.path.join(install_dir, item1)
844                src_dir = os.path.join(local_dir, item1)
845                if os.path.isdir(src_dir):
846                    check_call(['mkdir', '-p', dest_dir])
847                    # Create symlinks.
848                    for item2 in os.listdir(src_dir):
849                        src_item = os.path.join(src_dir, item2)
850                        logging.info('Creating symlink from {} in {}'.format(
851                            src_item, dest_dir))
852                        os.symlink(src_item, os.path.join(dest_dir, item2))
853        else:
854            logging.info('Fetching local VNDK snapshot from {}'.format(
855                local_dir))
856            unzip_artifacts(local_dir)
857    else:
858        raise RuntimeError('Neither local nor remote fetch information given.')
859
860def get_args():
861    parser = argparse.ArgumentParser()
862    parser.add_argument(
863        'snapshot_version',
864        type=int,
865        help='Vendor snapshot version to install, e.g. "30".')
866    parser.add_argument(
867        '--image',
868        help=('Image whose snapshot is being updated (e.g., vendor, '
869              'recovery , ramdisk, host, etc.)'),
870        default='vendor')
871    parser.add_argument('--branch', help='Branch to pull build from.')
872    parser.add_argument('--build', help='Build number to pull.')
873    parser.add_argument('--target', help='Target to pull.')
874    parser.add_argument(
875        '--local',
876        help=('Fetch local vendor snapshot artifacts from specified local '
877              'directory instead of Android Build server. '
878              'Example: --local /path/to/local/dir'))
879    parser.add_argument(
880        '--symlink',
881        action='store_true',
882        help='Use symlinks instead of unzipping vendor snapshot zip')
883    parser.add_argument(
884        '--install-dir',
885        required=True,
886        help=(
887            'Base directory to which vendor snapshot artifacts are installed. '
888            'Example: --install-dir vendor/<company name>/vendor_snapshot/v30'))
889    parser.add_argument(
890        '--overwrite',
891        action='store_true',
892        help=(
893            'If provided, does not ask before overwriting the install-dir.'))
894    parser.add_argument(
895        '--check-module-usage',
896        action='store_true',
897        help='Check which modules are used.')
898    parser.add_argument(
899        '--check-module-usage-goal',
900        action='append',
901        help='Goal(s) for which --check-module-usage is calculated.')
902    parser.add_argument(
903        '--check-module-usage-ninja-file',
904        help='Ninja file for which --check-module-usage is calculated.')
905    parser.add_argument(
906        '--check-module-usage-output',
907        help='File to which to write the check-module-usage results.')
908    parser.add_argument(
909        '--vndk-dir',
910        help='Path to installed vndk snapshot directory. Needed to retrieve '
911             'the list of VNDK. prebuilts/vndk/v{ver} will be used by default.')
912
913    parser.add_argument(
914        '-v',
915        '--verbose',
916        action='count',
917        default=0,
918        help='Increase output verbosity, e.g. "-v", "-vv".')
919    return parser.parse_args()
920
921
922def main():
923    """Program entry point."""
924    args = get_args()
925
926    host_image = args.image == 'host'
927    verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
928    verbosity = min(args.verbose, 2)
929    logging.basicConfig(
930        format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
931        level=verbose_map[verbosity])
932
933    if not args.install_dir:
934        raise ValueError('Please provide --install-dir option.')
935    install_dir = os.path.expanduser(args.install_dir)
936
937    if args.check_module_usage:
938        ninja_binary = './prebuilts/build-tools/linux-x86/bin/ninja'
939
940        if not args.check_module_usage_goal:
941            raise ValueError('Please provide --check-module-usage-goal option.')
942        if not args.check_module_usage_ninja_file:
943            raise ValueError(
944                'Please provide --check-module-usage-ninja-file option.')
945        if not args.check_module_usage_output:
946            raise ValueError(
947                'Please provide --check-module-usage-output option.')
948
949        if host_image:
950            check_host_usage(install_dir, ninja_binary,
951                             args.check_module_usage_ninja_file,
952                             args.check_module_usage_goal,
953                             args.check_module_usage_output)
954        else:
955            check_module_usage(install_dir, ninja_binary, args.image,
956                               args.check_module_usage_ninja_file,
957                               args.check_module_usage_goal,
958                               args.check_module_usage_output)
959        return
960
961    local = None
962    if args.local:
963        local = os.path.expanduser(args.local)
964
965    if local:
966        if args.build or args.branch or args.target:
967            raise ValueError(
968                'When --local option is set, --branch, --build or --target cannot be '
969                'specified.')
970        elif not os.path.isdir(local):
971            raise RuntimeError(
972                'The specified local directory, {}, does not exist.'.format(
973                    local))
974    else:
975        if not (args.build and args.branch and args.target):
976            raise ValueError(
977                'Please provide --branch, --build and --target. Or set --local '
978                'option.')
979
980    snapshot_version = args.snapshot_version
981
982    if os.path.exists(install_dir):
983        def remove_dir():
984            logging.info('Removing {}'.format(install_dir))
985            check_call(['rm', '-rf', install_dir])
986        if args.overwrite:
987            remove_dir()
988        else:
989            resp = input('Directory {} already exists. IT WILL BE REMOVED.\n'
990                         'Are you sure? (yes/no): '.format(install_dir))
991            if resp == 'yes':
992                remove_dir()
993            elif resp == 'no':
994                logging.info('Cancelled snapshot install.')
995                return
996            else:
997                raise ValueError('Did not understand: ' + resp)
998    check_call(['mkdir', '-p', install_dir])
999
1000    if args.vndk_dir:
1001        vndk_dir = os.path.expanduser(args.vndk_dir)
1002    else:
1003        vndk_dir = 'prebuilts/vndk/v%d' % snapshot_version
1004        logging.debug('Using %s for vndk directory' % vndk_dir)
1005
1006    install_artifacts(
1007        image=args.image,
1008        branch=args.branch,
1009        build=args.build,
1010        target=args.target,
1011        local_dir=local,
1012        symlink=args.symlink,
1013        install_dir=install_dir)
1014
1015    if host_image:
1016        gen_host_bp_file(install_dir, snapshot_version)
1017
1018    else:
1019        gen_bp_files(args.image, vndk_dir, install_dir, snapshot_version)
1020
1021if __name__ == '__main__':
1022    main()
1023