#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (c) 2021 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import argparse sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from loader import subsystem_info # noqa: E402 from loader import platforms_loader # noqa: E402 from loader import generate_targets_gn # noqa: E402 from loader import load_ohos_build # noqa: E402 from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 def _load_component_dist(source_root_dir, target_os, target_cpu): _parts_variants_info = {} _dir = "component_dist/{}-{}/packages_to_install".format( target_os, target_cpu) _file_name = "dist_parts_info.json" _dist_parts_info_file = os.path.join(source_root_dir, _dir, _file_name) if not os.path.exists(_dist_parts_info_file): # If the file does not exist, do nothing and return return _parts_variants_info _parts_info = read_json_file(_dist_parts_info_file) if _parts_info is None: raise Exception("read file '{}' failed.".format(_dist_parts_info_file)) for _part_info in _parts_info: origin_part_name = _part_info.get('origin_part_name') if origin_part_name in _parts_variants_info: variants = _parts_variants_info.get(origin_part_name) else: variants = [] _variant_name = _part_info.get('variant_name') variants.append(_variant_name) _parts_variants_info[origin_part_name] = variants return _parts_variants_info def _get_real_part_name(original_part_name, current_platform, parts_variants): part_info = parts_variants.get(original_part_name) if part_info is None: return None, None if current_platform in part_info and current_platform != 'phone': real_name = '{}_{}'.format(original_part_name, current_platform) else: real_name = original_part_name return real_name, original_part_name def _get_platforms_all_parts(source_root_dir, target_os, target_cpu, all_parts, build_platforms, parts_variants): _dist_parts_variants = _load_component_dist(source_root_dir, target_os, target_cpu) target_platform_parts = {} for _platform, _parts in all_parts.items(): if _platform not in build_platforms: continue part_name_info = {} for part_def in _parts: real_name, original_name = _get_real_part_name( part_def, _platform, parts_variants) if real_name is None: # find this from component_dist real_name, original_name = _get_real_part_name( part_def, _platform, _dist_parts_variants) if real_name is None: continue part_name_info[real_name] = original_name target_platform_parts[_platform] = part_name_info return target_platform_parts def _get_platforms_all_stubs(source_root_dir, target_os, target_cpu, all_stubs, build_platforms, parts_variants): _dist_parts_variants = _load_component_dist(source_root_dir, target_os, target_cpu) platform_stubs = {} for _platform, _part_names in all_stubs.items(): if _platform not in build_platforms: continue stub_parts_from_src = [] stub_parts_from_dist = [] for part_name in _part_names: real_name, original_name = _get_real_part_name( part_name, _platform, parts_variants) # real_name=None means part_name doesn't exist in source tree, # use binary in component_dist then. if real_name is None: # find this from component_dist real_name, original_name = _get_real_part_name( part_name, _platform, _dist_parts_variants) if real_name is None: continue else: stub_sources = os.path.join( source_root_dir, "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt" # noqa: E501 .format(target_os, target_cpu, real_name)) stub_parts_from_dist.append('"{}"'.format(stub_sources)) else: stub_parts_from_src.append(real_name) platform_stubs[_platform] = { "src": stub_parts_from_src, "dist": stub_parts_from_dist, } return platform_stubs def _get_platforms_parts(src_parts_targets, target_platform_parts): platforms_parts = {} src_all_parts = src_parts_targets.keys() for _platform, _all_parts in target_platform_parts.items(): src_parts_list = [] no_src_parts_list = [] for _part in _all_parts.keys(): if _part in src_all_parts: src_parts_list.append(_part) else: no_src_parts_list.append(_part) _data = { 'src_parts': src_parts_list, 'no_src_parts': no_src_parts_list } platforms_parts[_platform] = _data return platforms_parts def _get_parts_by_platform(target_platform_parts): parts_info = {} if 'phone' in target_platform_parts: phone_parts_list = target_platform_parts.get('phone').keys() else: phone_parts_list = [] for _platform, _parts_info in target_platform_parts.items(): base_parts_list = [] curr_parts_list = [] for _real_name, _original_name in _parts_info.items(): if _real_name in phone_parts_list: base_parts_list.append(_real_name) elif _original_name in phone_parts_list: base_parts_list.append(_real_name) else: curr_parts_list.append(_real_name) result_data = { "base_parts_list": base_parts_list, "curr_parts_list": curr_parts_list } parts_info[_platform] = result_data return parts_info def _check_parts_config_info(parts_config_info): if not ('parts_info' in parts_config_info and 'subsystem_parts' in parts_config_info and 'parts_variants' in parts_config_info and 'parts_kits_info' in parts_config_info and 'parts_inner_kits_info' in parts_config_info and 'parts_targets' in parts_config_info): raise Exception("Loading ohos.build information is incorrect.") def _get_required_build_parts_list(target_platform_parts): parts_set = set() for _parts_list in target_platform_parts.values(): parts_set.update(_parts_list) return list(parts_set) def _get_required_build_targets(parts_targets, target_platform_parts): required_build_targets = {} _parts_list = _get_required_build_parts_list(target_platform_parts) for _p_name, _info in parts_targets.items(): if _p_name not in _parts_list: continue required_build_targets[_p_name] = _info return required_build_targets def _get_auto_install_list(parts_path_info): auto_install_part_list = [] for part, path in parts_path_info.items(): if str(path).startswith("drivers/interface") or \ str(path).startswith("third_party"): auto_install_part_list.append(part) return auto_install_part_list def _get_parts_src_list(required_parts_targets, parts_info): parts_name_map = {} for _list in parts_info.values(): for _info in _list: parts_name_map[_info.get('part_name')] = _info.get( 'origin_part_name') _src_set = set() for _name in required_parts_targets.keys(): _origin_name = parts_name_map.get(_name) if _origin_name is None: continue _src_set.add(_origin_name) return list(_src_set) def _check_product_part_feature(parts_info, product_preloader_dir): _preloader_feature_file = os.path.join(product_preloader_dir, 'features.json') _preloader_feature_info = read_json_file(_preloader_feature_file) part_to_feature = _preloader_feature_info.get('part_to_feature') for key, vals in part_to_feature.items(): part = parts_info.get(key) if part is None: continue _p_info = part[0] def_feature_list = _p_info.get('feature_list') if not def_feature_list: continue for _f_name in vals: if _f_name not in def_feature_list: raise Exception( "The product use a feature that is not supported" " by this part, part_name='{}', feature='{}'".format( key, _f_name)) def _check_args(args, source_root_dir): print('args:', args) if 'gn_root_out_dir' not in args: raise Exception("args gn_root_out_dir is required.") if 'platforms_config_file' not in args: raise Exception("args platforms_config_file is required.") if 'subsystem_config_file' not in args: raise Exception("args subsystem_config_file is required.") gn_root_out_dir = args.gn_root_out_dir if gn_root_out_dir.startswith('/'): args.gn_root_out_dir = os.path.relpath(args.gn_root_out_dir, source_root_dir) else: _real_out_dir = os.path.realpath(gn_root_out_dir) if not _real_out_dir.startswith(source_root_dir): raise Exception("args gn_root_out_dir is incorrect.") def syscap_sort(syscap): return syscap['component'] def generate_syscap_files(parts_config_info, target_platform_parts, pre_syscap_info_path, system_path): syscap_product_dict = read_json_file(os.path.join(pre_syscap_info_path, "syscap.json")) target_parts_list = _get_required_build_parts_list(target_platform_parts) syscap_info_list = parts_config_info.get('syscap_info') target_syscap_with_part_name_list = [] target_syscap_list = [] target_syscap_for_init_list = [] all_syscap_list = [] for syscap in syscap_info_list: if syscap['component'] not in target_parts_list: continue if 'syscap' not in syscap or syscap['syscap'] == None or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: continue for syscap_string in syscap['syscap']: all_syscap_list.append(syscap_string.split('=')[0].strip()) for key, value in syscap_product_dict['part_to_syscap'].items(): for syscap in value: if syscap not in all_syscap_list: raise Exception( "In config.json of part [{}],the syscap[{}] is incorrect, \ please check the syscap name".format(key, syscap)) for syscap in syscap_info_list: remove_list = [] if syscap['component'] not in target_parts_list: continue if 'syscap' not in syscap or syscap['syscap'] == None or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: continue for syscap_string in syscap['syscap']: if syscap_string.startswith("SystemCapability.") == True: target_syscap_init_str = "const." syscap_name = syscap_string.split('=')[0].strip() all_syscap_product = syscap_product_dict['syscap'] if syscap_name in all_syscap_product and not all_syscap_product[syscap_name]: remove_list.append(syscap_string) continue elif syscap_name in all_syscap_product and all_syscap_product[syscap_name]: target_syscap_init_str += syscap_name + '=true\n' else: if syscap_string.endswith('true'): target_syscap_init_str += syscap_name + '=true\n' elif syscap_string.endswith('false'): remove_list.append(syscap_string) continue else: target_syscap_init_str += syscap_string + "=true\n" if target_syscap_init_str not in target_syscap_for_init_list: target_syscap_for_init_list.append(target_syscap_init_str) else: raise Exception("""In bundle.json of part [{}], The syscap string [{}] is incorrect, need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string)) for remove_str in remove_list: syscap['syscap'].remove(remove_str) for i in range(len(syscap['syscap'])): if syscap['syscap'][i].endswith('true') or syscap['syscap'][i].endswith('false'): syscap['syscap'][i] = syscap['syscap'][i].split('=')[0].strip() syscap['syscap'].sort() target_syscap_with_part_name_list.append(syscap) target_syscap_list.extend(syscap['syscap']) # Generate SystemCapability.json & syscap.json & syscap.para target_syscap_list.sort() syscap_info_dict = read_json_file(os.path.join(pre_syscap_info_path, "SystemCapability.json")) syscap_info_dict.update({'syscap':{'os':target_syscap_list}}) system_etc_path = os.path.join(system_path, "etc/") if not os.path.exists(system_path): os.mkdir(system_path) if not os.path.exists(system_etc_path): os.mkdir(system_etc_path) syscap_info_json = os.path.join(system_etc_path, "SystemCapability.json") write_json_file(syscap_info_json, syscap_info_dict) target_syscap_with_part_name_list.sort(key = syscap_sort) syscap_info_with_part_name_file = os.path.join(system_etc_path, "syscap.json") write_json_file(syscap_info_with_part_name_file, {'components': target_syscap_with_part_name_list}) if not os.path.exists(os.path.join(system_etc_path, "param/")): os.mkdir(os.path.join(system_etc_path, "param/")) target_syscap_for_init_file = os.path.join(system_etc_path, "param/syscap.para") f = open(target_syscap_for_init_file, "w") f.writelines(target_syscap_for_init_list) f.close() def load(args): source_root_dir = args.source_root_dir _check_args(args, source_root_dir) config_output_relpath = os.path.join(args.gn_root_out_dir, 'build_configs') # loading subsystem info, scan src dir and get subsystem ohos.build _subsystem_info = subsystem_info.get_subsystem_info( args.subsystem_config_file, args.example_subsystem_file, source_root_dir, config_output_relpath, args.os_level) target_arch = '{}_{}'.format(args.target_os, args.target_cpu) # loading platforms config _platforms_info = platforms_loader.get_platforms_info( args.platforms_config_file, source_root_dir, args.gn_root_out_dir, target_arch, config_output_relpath, args.scalable_build) # get build platforms list toolchain_to_variant_dict = _platforms_info.get('variant_toolchain_info') variant_toolchains = toolchain_to_variant_dict.get('platform_toolchain') _all_platforms = variant_toolchains.keys() if args.build_platform_name == 'all': build_platforms = _all_platforms elif args.build_platform_name in _all_platforms: build_platforms = [args.build_platform_name] else: raise Exception( "The target_platform is incorrect, only allows [{}].".format( ', '.join(_all_platforms))) # loading ohos.build and gen part variant info parts_config_info = load_ohos_build.get_parts_info( source_root_dir, config_output_relpath, _subsystem_info, variant_toolchains, target_arch, args.ignore_api_check, args.exclusion_modules_config_file, args.load_test_config, args.build_xts) # check parts_config_info _check_parts_config_info(parts_config_info) parts_variants = parts_config_info.get('parts_variants') parts_targets = parts_config_info.get('parts_targets') parts_info = parts_config_info.get('parts_info') config_output_dir = os.path.join(source_root_dir, config_output_relpath) # target_platforms_parts.json target_platform_parts = _get_platforms_all_parts( source_root_dir, args.target_os, args.target_cpu, _platforms_info.get('all_parts'), build_platforms, parts_variants) target_platform_parts_file = os.path.join(config_output_dir, "target_platforms_parts.json") write_json_file(target_platform_parts_file, target_platform_parts, check_changes=True) # {platform}_system_capabilities.json # we assume that platform and devicetype are the same. for platform in build_platforms: platform_parts = target_platform_parts.get(platform) platform_capabilities = [] for _, origin in platform_parts.items(): # parts_info.get() might be None if the part is a binary package all_parts_variants = parts_info.get(origin) if all_parts_variants is None: continue part = all_parts_variants[0] if part.get('system_capabilities'): entry = part.get('system_capabilities') if len(entry) > 0: platform_capabilities.extend(entry) platform_part_json_file = os.path.join( config_output_dir, "{0}_system_capabilities.json".format(platform)) write_json_file(platform_part_json_file, sorted(platform_capabilities), check_changes=True) target_platform_stubs = _get_platforms_all_stubs( source_root_dir, args.target_os, args.target_cpu, _platforms_info.get('all_stubs'), build_platforms, parts_variants) generate_targets_gn.gen_stub_targets( parts_config_info.get('parts_kits_info'), target_platform_stubs, config_output_dir) # platforms_parts_by_src.json platforms_parts_by_src = _get_platforms_parts(parts_targets, target_platform_parts) platforms_parts_by_src_file = os.path.join(source_root_dir, config_output_relpath, "platforms_parts_by_src.json") write_json_file(platforms_parts_by_src_file, platforms_parts_by_src, check_changes=True) required_parts_targets = _get_required_build_targets( parts_targets, target_platform_parts) generate_targets_gn.gen_targets_gn(required_parts_targets, config_output_dir) _phony_target = parts_config_info.get('phony_target') required_phony_targets = _get_required_build_targets( _phony_target, target_platform_parts) generate_targets_gn.gen_phony_targets(required_phony_targets, config_output_dir) # required_parts_targets.json build_targets_info_file = os.path.join(config_output_dir, "required_parts_targets.json") write_json_file(build_targets_info_file, required_parts_targets) # required_parts_targets_list.json build_targets_list_file = os.path.join(config_output_dir, "required_parts_targets_list.json") write_json_file(build_targets_list_file, list(required_parts_targets.values())) # parts src flag file parts_src_flag_file = os.path.join(config_output_dir, "parts_src_flag.json") write_json_file(parts_src_flag_file, _get_parts_src_list(required_parts_targets, parts_info), check_changes=True) # write auto install part file auto_install_list = _get_auto_install_list(parts_config_info.get("parts_path_info")) auto_install_list_file = os.path.join(config_output_dir, "auto_install_parts.json") write_json_file(auto_install_list_file, auto_install_list) # write platforms_list.gni platforms_list_gni_file = os.path.join(config_output_dir, "platforms_list.gni") _platforms = set(build_platforms) _gni_file_content = [] _gni_file_content.append('target_platform_list = [') _gni_file_content.append(' "{}"'.format('",\n "'.join(_platforms))) _gni_file_content.append(']') _gni_file_content.append('kits_platform_list = [') _gni_file_content.append(' "{}",'.format('",\n "'.join(_platforms))) if 'phone' not in build_platforms: _gni_file_content.append(' "phone"') _gni_file_content.append(']') write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) # parts_different_info.json # Generate parts differences in different platforms, using phone as base. parts_different_info = _get_parts_by_platform(target_platform_parts) parts_different_info_file = os.path.join(config_output_dir, "parts_different_info.json") write_json_file(parts_different_info_file, parts_different_info, check_changes=True) # for testfwk infos_for_testfwk_file = os.path.join(config_output_dir, "infos_for_testfwk.json") _output_infos_for_testfwk(parts_config_info, target_platform_parts, infos_for_testfwk_file) # check part feature _check_product_part_feature(parts_info, os.path.dirname(args.platforms_config_file)) pre_syscap_info_path = os.path.dirname(args.platforms_config_file) system_path = os.path.join(source_root_dir, os.path.join(os.path.dirname(args.platforms_config_file), "system/")) generate_syscap_files(parts_config_info, target_platform_parts, pre_syscap_info_path, system_path) def _output_infos_by_platform(part_name_infos, parts_info_dict): required_parts = {} subsystem_infos = {} for part_name, origin_part_name in part_name_infos.items(): part_info = parts_info_dict.get(part_name) if part_info is None: continue if origin_part_name != part_info.get('origin_part_name'): raise Exception("part configuration is incorrect.") required_parts[origin_part_name] = part_info _subsystem_name = part_info.get('subsystem_name') if _subsystem_name in subsystem_infos: p_list = subsystem_infos.get(_subsystem_name) else: p_list = [] p_list.append(origin_part_name) subsystem_infos[_subsystem_name] = p_list result = {} result['subsystem_infos'] = subsystem_infos result['part_infos'] = required_parts return result def _output_infos_for_testfwk(parts_config_info, target_platform_parts, infos_for_testfwk_file): parts_info = parts_config_info.get('parts_info') parts_info_dict = {} for _part_name, _parts in parts_info.items(): for _info in _parts: parts_info_dict[_info.get('part_name')] = _info _output_infos = {} for _platform, _parts in target_platform_parts.items(): result = _output_infos_by_platform(_parts, parts_info_dict) _output_infos[_platform] = result write_json_file(infos_for_testfwk_file, _output_infos, check_changes=True) def main(): parser = argparse.ArgumentParser() parser.add_argument('--platforms-config-file', required=True) parser.add_argument('--subsystem-config-file', required=True) parser.add_argument('--example-subsystem-file', required=False) parser.add_argument('--exclusion-modules-config-file', required=False) parser.add_argument('--source-root-dir', required=True) parser.add_argument('--gn-root-out-dir', default='.') parser.add_argument('--build-platform-name', default='phone') parser.add_argument('--build-xts', dest='build_xts', action='store_true') parser.set_defaults(build_xts=False) parser.add_argument('--load-test-config', action='store_true') parser.add_argument('--target-os', default='ohos') parser.add_argument('--target-cpu', default='arm64') parser.add_argument('--os-level', default='standard') parser.add_argument('--ignore-api-check', nargs='*', default=[]) parser.add_argument('--scalable-build', action='store_true') parser.set_defaults(scalable_build=False) args = parser.parse_args() load(args) return 0 if __name__ == '__main__': sys.exit(main())