1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19 20sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 21from loader import subsystem_info # noqa: E402 22from loader import platforms_loader # noqa: E402 23from loader import generate_targets_gn # noqa: E402 24from loader import load_ohos_build # noqa: E402 25from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 26 27 28def _load_component_dist(source_root_dir, target_os, target_cpu): 29 _parts_variants_info = {} 30 _dir = "component_dist/{}-{}/packages_to_install".format( 31 target_os, target_cpu) 32 _file_name = "dist_parts_info.json" 33 _dist_parts_info_file = os.path.join(source_root_dir, _dir, _file_name) 34 if not os.path.exists(_dist_parts_info_file): 35 # If the file does not exist, do nothing and return 36 return _parts_variants_info 37 _parts_info = read_json_file(_dist_parts_info_file) 38 if _parts_info is None: 39 raise Exception("read file '{}' failed.".format(_dist_parts_info_file)) 40 for _part_info in _parts_info: 41 origin_part_name = _part_info.get('origin_part_name') 42 if origin_part_name in _parts_variants_info: 43 variants = _parts_variants_info.get(origin_part_name) 44 else: 45 variants = [] 46 _variant_name = _part_info.get('variant_name') 47 variants.append(_variant_name) 48 _parts_variants_info[origin_part_name] = variants 49 return _parts_variants_info 50 51 52def _get_real_part_name(original_part_name, current_platform, parts_variants): 53 part_info = parts_variants.get(original_part_name) 54 if part_info is None: 55 return None, None 56 if current_platform in part_info and current_platform != 'phone': 57 real_name = '{}_{}'.format(original_part_name, current_platform) 58 else: 59 real_name = original_part_name 60 return real_name, original_part_name 61 62 63def _get_platforms_all_parts(source_root_dir, target_os, target_cpu, all_parts, 64 build_platforms, parts_variants): 65 _dist_parts_variants = _load_component_dist(source_root_dir, target_os, 66 target_cpu) 67 target_platform_parts = {} 68 for _platform, _parts in all_parts.items(): 69 if _platform not in build_platforms: 70 continue 71 part_name_info = {} 72 for part_def in _parts: 73 real_name, original_name = _get_real_part_name( 74 part_def, _platform, parts_variants) 75 if real_name is None: 76 # find this from component_dist 77 real_name, original_name = _get_real_part_name( 78 part_def, _platform, _dist_parts_variants) 79 if real_name is None: 80 continue 81 part_name_info[real_name] = original_name 82 target_platform_parts[_platform] = part_name_info 83 return target_platform_parts 84 85 86def _get_platforms_all_stubs(source_root_dir, target_os, target_cpu, all_stubs, 87 build_platforms, parts_variants): 88 _dist_parts_variants = _load_component_dist(source_root_dir, target_os, 89 target_cpu) 90 platform_stubs = {} 91 for _platform, _part_names in all_stubs.items(): 92 if _platform not in build_platforms: 93 continue 94 stub_parts_from_src = [] 95 stub_parts_from_dist = [] 96 for part_name in _part_names: 97 real_name, original_name = _get_real_part_name( 98 part_name, _platform, parts_variants) 99 # real_name=None means part_name doesn't exist in source tree, 100 # use binary in component_dist then. 101 if real_name is None: 102 # find this from component_dist 103 real_name, original_name = _get_real_part_name( 104 part_name, _platform, _dist_parts_variants) 105 if real_name is None: 106 continue 107 else: 108 stub_sources = os.path.join( 109 source_root_dir, 110 "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt" # noqa: E501 111 .format(target_os, target_cpu, real_name)) 112 stub_parts_from_dist.append('"{}"'.format(stub_sources)) 113 else: 114 stub_parts_from_src.append(real_name) 115 platform_stubs[_platform] = { 116 "src": stub_parts_from_src, 117 "dist": stub_parts_from_dist, 118 } 119 return platform_stubs 120 121 122def _get_platforms_parts(src_parts_targets, target_platform_parts): 123 platforms_parts = {} 124 src_all_parts = src_parts_targets.keys() 125 for _platform, _all_parts in target_platform_parts.items(): 126 src_parts_list = [] 127 no_src_parts_list = [] 128 for _part in _all_parts.keys(): 129 if _part in src_all_parts: 130 src_parts_list.append(_part) 131 else: 132 no_src_parts_list.append(_part) 133 _data = { 134 'src_parts': src_parts_list, 135 'no_src_parts': no_src_parts_list 136 } 137 platforms_parts[_platform] = _data 138 return platforms_parts 139 140 141def _get_parts_by_platform(target_platform_parts): 142 parts_info = {} 143 if 'phone' in target_platform_parts: 144 phone_parts_list = target_platform_parts.get('phone').keys() 145 else: 146 phone_parts_list = [] 147 for _platform, _parts_info in target_platform_parts.items(): 148 base_parts_list = [] 149 curr_parts_list = [] 150 for _real_name, _original_name in _parts_info.items(): 151 if _real_name in phone_parts_list: 152 base_parts_list.append(_real_name) 153 elif _original_name in phone_parts_list: 154 base_parts_list.append(_real_name) 155 else: 156 curr_parts_list.append(_real_name) 157 result_data = { 158 "base_parts_list": base_parts_list, 159 "curr_parts_list": curr_parts_list 160 } 161 parts_info[_platform] = result_data 162 return parts_info 163 164 165def _check_parts_config_info(parts_config_info): 166 if not ('parts_info' in parts_config_info and 'subsystem_parts' 167 in parts_config_info and 'parts_variants' in parts_config_info 168 and 'parts_kits_info' in parts_config_info 169 and 'parts_inner_kits_info' in parts_config_info 170 and 'parts_targets' in parts_config_info): 171 raise Exception("Loading ohos.build information is incorrect.") 172 173 174def _get_required_build_parts_list(target_platform_parts): 175 parts_set = set() 176 for _parts_list in target_platform_parts.values(): 177 parts_set.update(_parts_list) 178 return list(parts_set) 179 180 181def _get_required_build_targets(parts_targets, target_platform_parts): 182 required_build_targets = {} 183 _parts_list = _get_required_build_parts_list(target_platform_parts) 184 for _p_name, _info in parts_targets.items(): 185 if _p_name not in _parts_list: 186 continue 187 required_build_targets[_p_name] = _info 188 return required_build_targets 189 190 191def _get_parts_src_list(required_parts_targets, parts_info): 192 parts_name_map = {} 193 for _list in parts_info.values(): 194 for _info in _list: 195 parts_name_map[_info.get('part_name')] = _info.get( 196 'origin_part_name') 197 _src_set = set() 198 for _name in required_parts_targets.keys(): 199 _origin_name = parts_name_map.get(_name) 200 if _origin_name is None: 201 continue 202 _src_set.add(_origin_name) 203 return list(_src_set) 204 205 206def _check_product_part_feature(parts_info, product_preloader_dir): 207 _preloader_feature_file = os.path.join(product_preloader_dir, 208 'features.json') 209 _preloader_feature_info = read_json_file(_preloader_feature_file) 210 part_to_feature = _preloader_feature_info.get('part_to_feature') 211 for key, vals in part_to_feature.items(): 212 part = parts_info.get(key) 213 if part is None: 214 continue 215 _p_info = part[0] 216 def_feature_list = _p_info.get('feature_list') 217 if not def_feature_list: 218 continue 219 for _f_name in vals: 220 if _f_name not in def_feature_list: 221 raise Exception( 222 "The product use a feature that is not supported" 223 " by this part, part_name='{}', feature='{}'".format( 224 key, _f_name)) 225 226 227def _check_args(args, source_root_dir): 228 print('args:', args) 229 if 'gn_root_out_dir' not in args: 230 raise Exception("args gn_root_out_dir is required.") 231 if 'platforms_config_file' not in args: 232 raise Exception("args platforms_config_file is required.") 233 if 'subsystem_config_file' not in args: 234 raise Exception("args subsystem_config_file is required.") 235 gn_root_out_dir = args.gn_root_out_dir 236 if gn_root_out_dir.startswith('/'): 237 args.gn_root_out_dir = os.path.relpath(args.gn_root_out_dir, 238 source_root_dir) 239 else: 240 _real_out_dir = os.path.realpath(gn_root_out_dir) 241 if not _real_out_dir.startswith(source_root_dir): 242 raise Exception("args gn_root_out_dir is incorrect.") 243 244def syscap_sort(syscap): 245 return syscap['component'] 246 247def generate_syscap_files(parts_config_info, target_platform_parts, pre_syscap_info_path, system_path): 248 target_parts_list = _get_required_build_parts_list(target_platform_parts) 249 syscap_info_list = parts_config_info.get('syscap_info') 250 target_syscap_with_part_name_list = [] 251 target_syscap_list = [] 252 target_syscap_for_init_list = [] 253 for syscap in syscap_info_list: 254 if syscap['component'] not in target_parts_list: 255 continue 256 if 'syscap' not in syscap or syscap['syscap'] == None or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 257 continue 258 for syscap_string in syscap['syscap']: 259 if syscap_string.startswith("SystemCapability.") == True: 260 target_syscap_for_init_list.append("const." + syscap_string + "=true\n") 261 else: 262 raise Exception("""In bundle.json of part [{}], The syscap string [{}] is incorrect, 263 need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string)) 264 syscap['syscap'].sort() 265 target_syscap_with_part_name_list.append(syscap) 266 target_syscap_list.extend(syscap['syscap']) 267 268 # Generate SystemCapability.json & syscap.json & syscap.para 269 target_syscap_list.sort() 270 syscap_info_dict = read_json_file(pre_syscap_info_path) 271 syscap_info_dict.update({'syscap':{'os':target_syscap_list}}) 272 system_etc_path = os.path.join(system_path, "etc/") 273 if not os.path.exists(system_path): 274 os.mkdir(system_path) 275 if not os.path.exists(system_etc_path): 276 os.mkdir(system_etc_path) 277 syscap_info_json = os.path.join(system_etc_path, "SystemCapability.json") 278 write_json_file(syscap_info_json, syscap_info_dict) 279 target_syscap_with_part_name_list.sort(key = syscap_sort) 280 syscap_info_with_part_name_file = os.path.join(system_etc_path, "syscap.json") 281 write_json_file(syscap_info_with_part_name_file, {'components': target_syscap_with_part_name_list}) 282 if not os.path.exists(os.path.join(system_etc_path, "param/")): 283 os.mkdir(os.path.join(system_etc_path, "param/")) 284 target_syscap_for_init_file = os.path.join(system_etc_path, "param/syscap.para") 285 f = open(target_syscap_for_init_file, "w") 286 f.writelines(target_syscap_for_init_list) 287 f.close() 288 289def load(args): 290 source_root_dir = args.source_root_dir 291 _check_args(args, source_root_dir) 292 config_output_relpath = os.path.join(args.gn_root_out_dir, 'build_configs') 293 294 # loading subsystem info, scan src dir and get subsystem ohos.build 295 _subsystem_info = subsystem_info.get_subsystem_info( 296 args.subsystem_config_file, args.example_subsystem_file, 297 source_root_dir, config_output_relpath, args.os_level) 298 299 target_arch = '{}_{}'.format(args.target_os, args.target_cpu) 300 # loading platforms config 301 _platforms_info = platforms_loader.get_platforms_info( 302 args.platforms_config_file, source_root_dir, args.gn_root_out_dir, 303 target_arch, config_output_relpath, args.scalable_build) 304 305 # get build platforms list 306 toolchain_to_variant_dict = _platforms_info.get('variant_toolchain_info') 307 variant_toolchains = toolchain_to_variant_dict.get('platform_toolchain') 308 _all_platforms = variant_toolchains.keys() 309 310 if args.build_platform_name == 'all': 311 build_platforms = _all_platforms 312 elif args.build_platform_name in _all_platforms: 313 build_platforms = [args.build_platform_name] 314 else: 315 raise Exception( 316 "The target_platform is incorrect, only allows [{}].".format( 317 ', '.join(_all_platforms))) 318 319 # loading ohos.build and gen part variant info 320 parts_config_info = load_ohos_build.get_parts_info( 321 source_root_dir, config_output_relpath, _subsystem_info, 322 variant_toolchains, target_arch, args.ignore_api_check, args.build_xts) 323 # check parts_config_info 324 _check_parts_config_info(parts_config_info) 325 parts_variants = parts_config_info.get('parts_variants') 326 parts_targets = parts_config_info.get('parts_targets') 327 parts_info = parts_config_info.get('parts_info') 328 329 config_output_dir = os.path.join(source_root_dir, config_output_relpath) 330 331 # target_platforms_parts.json 332 target_platform_parts = _get_platforms_all_parts( 333 source_root_dir, args.target_os, args.target_cpu, 334 _platforms_info.get('all_parts'), build_platforms, parts_variants) 335 target_platform_parts_file = os.path.join(config_output_dir, 336 "target_platforms_parts.json") 337 write_json_file(target_platform_parts_file, 338 target_platform_parts, 339 check_changes=True) 340 341 # {platform}_system_capabilities.json 342 # we assume that platform and devicetype are the same. 343 for platform in build_platforms: 344 platform_parts = target_platform_parts.get(platform) 345 platform_capabilities = [] 346 for _, origin in platform_parts.items(): 347 # parts_info.get() might be None if the part is a binary package 348 all_parts_variants = parts_info.get(origin) 349 if all_parts_variants is None: 350 continue 351 part = all_parts_variants[0] 352 if part.get('system_capabilities'): 353 entry = part.get('system_capabilities') 354 if len(entry) > 0: 355 platform_capabilities.extend(entry) 356 platform_part_json_file = os.path.join( 357 config_output_dir, "{0}_system_capabilities.json".format(platform)) 358 write_json_file(platform_part_json_file, 359 sorted(platform_capabilities), 360 check_changes=True) 361 362 target_platform_stubs = _get_platforms_all_stubs( 363 source_root_dir, args.target_os, args.target_cpu, 364 _platforms_info.get('all_stubs'), build_platforms, parts_variants) 365 generate_targets_gn.gen_stub_targets( 366 parts_config_info.get('parts_kits_info'), target_platform_stubs, 367 config_output_dir) 368 369 # platforms_parts_by_src.json 370 platforms_parts_by_src = _get_platforms_parts(parts_targets, 371 target_platform_parts) 372 platforms_parts_by_src_file = os.path.join(source_root_dir, 373 config_output_relpath, 374 "platforms_parts_by_src.json") 375 write_json_file(platforms_parts_by_src_file, 376 platforms_parts_by_src, 377 check_changes=True) 378 379 required_parts_targets = _get_required_build_targets( 380 parts_targets, target_platform_parts) 381 generate_targets_gn.gen_targets_gn(required_parts_targets, 382 config_output_dir) 383 _phony_target = parts_config_info.get('phony_target') 384 required_phony_targets = _get_required_build_targets( 385 _phony_target, target_platform_parts) 386 generate_targets_gn.gen_phony_targets(required_phony_targets, 387 config_output_dir) 388 389 # required_parts_targets.json 390 build_targets_info_file = os.path.join(config_output_dir, 391 "required_parts_targets.json") 392 write_json_file(build_targets_info_file, required_parts_targets) 393 # required_parts_targets_list.json 394 build_targets_list_file = os.path.join(config_output_dir, 395 "required_parts_targets_list.json") 396 write_json_file(build_targets_list_file, 397 list(required_parts_targets.values())) 398 399 # parts src flag file 400 parts_src_flag_file = os.path.join(config_output_dir, 401 "parts_src_flag.json") 402 write_json_file(parts_src_flag_file, 403 _get_parts_src_list(required_parts_targets, parts_info), 404 check_changes=True) 405 406 # write platforms_list.gni 407 platforms_list_gni_file = os.path.join(config_output_dir, 408 "platforms_list.gni") 409 _platforms = set(build_platforms) 410 _gni_file_content = [] 411 _gni_file_content.append('target_platform_list = [') 412 _gni_file_content.append(' "{}"'.format('",\n "'.join(_platforms))) 413 _gni_file_content.append(']') 414 _gni_file_content.append('kits_platform_list = [') 415 _gni_file_content.append(' "{}",'.format('",\n "'.join(_platforms))) 416 if 'phone' not in build_platforms: 417 _gni_file_content.append(' "phone"') 418 _gni_file_content.append(']') 419 write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) 420 421 # parts_different_info.json 422 # Generate parts differences in different platforms, using phone as base. 423 parts_different_info = _get_parts_by_platform(target_platform_parts) 424 parts_different_info_file = os.path.join(config_output_dir, 425 "parts_different_info.json") 426 write_json_file(parts_different_info_file, 427 parts_different_info, 428 check_changes=True) 429 # for testfwk 430 infos_for_testfwk_file = os.path.join(config_output_dir, 431 "infos_for_testfwk.json") 432 _output_infos_for_testfwk(parts_config_info, target_platform_parts, 433 infos_for_testfwk_file) 434 435 # check part feature 436 _check_product_part_feature(parts_info, 437 os.path.dirname(args.platforms_config_file)) 438 pre_syscap_info_path = os.path.join(os.path.dirname(args.platforms_config_file), "SystemCapability.json") 439 system_path = os.path.join(source_root_dir, os.path.join(os.path.dirname(args.platforms_config_file), "system/")) 440 generate_syscap_files(parts_config_info, target_platform_parts, pre_syscap_info_path, system_path) 441 442def _output_infos_by_platform(part_name_infos, parts_info_dict): 443 required_parts = {} 444 subsystem_infos = {} 445 for part_name, origin_part_name in part_name_infos.items(): 446 part_info = parts_info_dict.get(part_name) 447 if part_info is None: 448 continue 449 if origin_part_name != part_info.get('origin_part_name'): 450 raise Exception("part configuration is incorrect.") 451 required_parts[origin_part_name] = part_info 452 _subsystem_name = part_info.get('subsystem_name') 453 if _subsystem_name in subsystem_infos: 454 p_list = subsystem_infos.get(_subsystem_name) 455 else: 456 p_list = [] 457 p_list.append(origin_part_name) 458 subsystem_infos[_subsystem_name] = p_list 459 result = {} 460 result['subsystem_infos'] = subsystem_infos 461 result['part_infos'] = required_parts 462 return result 463 464def _output_infos_for_testfwk(parts_config_info, target_platform_parts, 465 infos_for_testfwk_file): 466 parts_info = parts_config_info.get('parts_info') 467 parts_info_dict = {} 468 for _part_name, _parts in parts_info.items(): 469 for _info in _parts: 470 parts_info_dict[_info.get('part_name')] = _info 471 472 _output_infos = {} 473 for _platform, _parts in target_platform_parts.items(): 474 result = _output_infos_by_platform(_parts, parts_info_dict) 475 _output_infos[_platform] = result 476 477 write_json_file(infos_for_testfwk_file, _output_infos, check_changes=True) 478 479 480def main(): 481 parser = argparse.ArgumentParser() 482 parser.add_argument('--platforms-config-file', required=True) 483 parser.add_argument('--subsystem-config-file', required=True) 484 parser.add_argument('--example-subsystem-file', required=False) 485 parser.add_argument('--source-root-dir', required=True) 486 parser.add_argument('--gn-root-out-dir', default='.') 487 parser.add_argument('--build-platform-name', default='phone') 488 parser.add_argument('--build-xts', dest='build_xts', action='store_true') 489 parser.set_defaults(build_xts=False) 490 parser.add_argument('--target-os', default='ohos') 491 parser.add_argument('--target-cpu', default='arm64') 492 parser.add_argument('--os-level', default='standard') 493 parser.add_argument('--ignore-api-check', nargs='*', default=[]) 494 495 parser.add_argument('--scalable-build', action='store_true') 496 parser.set_defaults(scalable_build=False) 497 args = parser.parse_args() 498 499 load(args) 500 return 0 501 502 503if __name__ == '__main__': 504 sys.exit(main()) 505