1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19 20sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 21from loader import subsystem_info # noqa: E402 22from loader import platforms_loader # noqa: E402 23from loader import generate_targets_gn # noqa: E402 24from loader import load_ohos_build # noqa: E402 25from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 26 27 28def _load_component_dist(source_root_dir, target_os, target_cpu): 29 _parts_variants_info = {} 30 _dir = "component_dist/{}-{}/packages_to_install".format( 31 target_os, target_cpu) 32 _file_name = "dist_parts_info.json" 33 _dist_parts_info_file = os.path.join(source_root_dir, _dir, _file_name) 34 if not os.path.exists(_dist_parts_info_file): 35 # If the file does not exist, do nothing and return 36 return _parts_variants_info 37 _parts_info = read_json_file(_dist_parts_info_file) 38 if _parts_info is None: 39 raise Exception("read file '{}' failed.".format(_dist_parts_info_file)) 40 for _part_info in _parts_info: 41 origin_part_name = _part_info.get('origin_part_name') 42 if origin_part_name in _parts_variants_info: 43 variants = _parts_variants_info.get(origin_part_name) 44 else: 45 variants = [] 46 _variant_name = _part_info.get('variant_name') 47 variants.append(_variant_name) 48 _parts_variants_info[origin_part_name] = variants 49 return _parts_variants_info 50 51 52def _get_real_part_name(original_part_name, current_platform, parts_variants): 53 part_info = parts_variants.get(original_part_name) 54 if part_info is None: 55 return None, None 56 if current_platform in part_info and current_platform != 'phone': 57 real_name = '{}_{}'.format(original_part_name, current_platform) 58 else: 59 real_name = original_part_name 60 return real_name, original_part_name 61 62 63def _get_platforms_all_parts(source_root_dir, target_os, target_cpu, all_parts, 64 build_platforms, parts_variants): 65 _dist_parts_variants = _load_component_dist(source_root_dir, target_os, 66 target_cpu) 67 target_platform_parts = {} 68 for _platform, _parts in all_parts.items(): 69 if _platform not in build_platforms: 70 continue 71 part_name_info = {} 72 for part_def in _parts: 73 real_name, original_name = _get_real_part_name( 74 part_def, _platform, parts_variants) 75 if real_name is None: 76 # find this from component_dist 77 real_name, original_name = _get_real_part_name( 78 part_def, _platform, _dist_parts_variants) 79 if real_name is None: 80 continue 81 part_name_info[real_name] = original_name 82 target_platform_parts[_platform] = part_name_info 83 return target_platform_parts 84 85 86def _get_platforms_all_stubs(source_root_dir, target_os, target_cpu, all_stubs, 87 build_platforms, parts_variants): 88 _dist_parts_variants = _load_component_dist(source_root_dir, target_os, 89 target_cpu) 90 platform_stubs = {} 91 for _platform, _part_names in all_stubs.items(): 92 if _platform not in build_platforms: 93 continue 94 stub_parts_from_src = [] 95 stub_parts_from_dist = [] 96 for part_name in _part_names: 97 real_name, original_name = _get_real_part_name( 98 part_name, _platform, parts_variants) 99 # real_name=None means part_name doesn't exist in source tree, 100 # use binary in component_dist then. 101 if real_name is None: 102 # find this from component_dist 103 real_name, original_name = _get_real_part_name( 104 part_name, _platform, _dist_parts_variants) 105 if real_name is None: 106 continue 107 else: 108 stub_sources = os.path.join( 109 source_root_dir, 110 "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt" # noqa: E501 111 .format(target_os, target_cpu, real_name)) 112 stub_parts_from_dist.append('"{}"'.format(stub_sources)) 113 else: 114 stub_parts_from_src.append(real_name) 115 platform_stubs[_platform] = { 116 "src": stub_parts_from_src, 117 "dist": stub_parts_from_dist, 118 } 119 return platform_stubs 120 121 122def _get_platforms_parts(src_parts_targets, target_platform_parts): 123 platforms_parts = {} 124 src_all_parts = src_parts_targets.keys() 125 for _platform, _all_parts in target_platform_parts.items(): 126 src_parts_list = [] 127 no_src_parts_list = [] 128 for _part in _all_parts.keys(): 129 if _part in src_all_parts: 130 src_parts_list.append(_part) 131 else: 132 no_src_parts_list.append(_part) 133 _data = { 134 'src_parts': src_parts_list, 135 'no_src_parts': no_src_parts_list 136 } 137 platforms_parts[_platform] = _data 138 return platforms_parts 139 140 141def _get_parts_by_platform(target_platform_parts): 142 parts_info = {} 143 if 'phone' in target_platform_parts: 144 phone_parts_list = target_platform_parts.get('phone').keys() 145 else: 146 phone_parts_list = [] 147 for _platform, _parts_info in target_platform_parts.items(): 148 base_parts_list = [] 149 curr_parts_list = [] 150 for _real_name, _original_name in _parts_info.items(): 151 if _real_name in phone_parts_list: 152 base_parts_list.append(_real_name) 153 elif _original_name in phone_parts_list: 154 base_parts_list.append(_real_name) 155 else: 156 curr_parts_list.append(_real_name) 157 result_data = { 158 "base_parts_list": base_parts_list, 159 "curr_parts_list": curr_parts_list 160 } 161 parts_info[_platform] = result_data 162 return parts_info 163 164 165def _check_parts_config_info(parts_config_info): 166 if not ('parts_info' in parts_config_info and 'subsystem_parts' 167 in parts_config_info and 'parts_variants' in parts_config_info 168 and 'parts_kits_info' in parts_config_info 169 and 'parts_inner_kits_info' in parts_config_info 170 and 'parts_targets' in parts_config_info): 171 raise Exception("Loading ohos.build information is incorrect.") 172 173 174def _get_required_build_parts_list(target_platform_parts): 175 parts_set = set() 176 for _parts_list in target_platform_parts.values(): 177 parts_set.update(_parts_list) 178 return list(parts_set) 179 180 181def _get_required_build_targets(parts_targets, target_platform_parts): 182 required_build_targets = {} 183 _parts_list = _get_required_build_parts_list(target_platform_parts) 184 for _p_name, _info in parts_targets.items(): 185 if _p_name not in _parts_list: 186 continue 187 required_build_targets[_p_name] = _info 188 return required_build_targets 189 190 191def _get_auto_install_list(parts_path_info): 192 auto_install_part_list = [] 193 for part, path in parts_path_info.items(): 194 if str(path).startswith("drivers/interface") or \ 195 str(path).startswith("third_party"): 196 auto_install_part_list.append(part) 197 return auto_install_part_list 198 199def _get_parts_src_list(required_parts_targets, parts_info): 200 parts_name_map = {} 201 for _list in parts_info.values(): 202 for _info in _list: 203 parts_name_map[_info.get('part_name')] = _info.get( 204 'origin_part_name') 205 _src_set = set() 206 for _name in required_parts_targets.keys(): 207 _origin_name = parts_name_map.get(_name) 208 if _origin_name is None: 209 continue 210 _src_set.add(_origin_name) 211 return list(_src_set) 212 213 214def _check_product_part_feature(parts_info, product_preloader_dir): 215 _preloader_feature_file = os.path.join(product_preloader_dir, 216 'features.json') 217 _preloader_feature_info = read_json_file(_preloader_feature_file) 218 part_to_feature = _preloader_feature_info.get('part_to_feature') 219 for key, vals in part_to_feature.items(): 220 part = parts_info.get(key) 221 if part is None: 222 continue 223 _p_info = part[0] 224 def_feature_list = _p_info.get('feature_list') 225 if not def_feature_list: 226 continue 227 for _f_name in vals: 228 if _f_name not in def_feature_list: 229 raise Exception( 230 "The product use a feature that is not supported" 231 " by this part, part_name='{}', feature='{}'".format( 232 key, _f_name)) 233 234 235def _check_args(args, source_root_dir): 236 print('args:', args) 237 if 'gn_root_out_dir' not in args: 238 raise Exception("args gn_root_out_dir is required.") 239 if 'platforms_config_file' not in args: 240 raise Exception("args platforms_config_file is required.") 241 if 'subsystem_config_file' not in args: 242 raise Exception("args subsystem_config_file is required.") 243 gn_root_out_dir = args.gn_root_out_dir 244 if gn_root_out_dir.startswith('/'): 245 args.gn_root_out_dir = os.path.relpath(args.gn_root_out_dir, 246 source_root_dir) 247 else: 248 _real_out_dir = os.path.realpath(gn_root_out_dir) 249 if not _real_out_dir.startswith(source_root_dir): 250 raise Exception("args gn_root_out_dir is incorrect.") 251 252def syscap_sort(syscap): 253 return syscap['component'] 254 255def generate_syscap_files(parts_config_info, target_platform_parts, pre_syscap_info_path, system_path): 256 syscap_product_dict = read_json_file(os.path.join(pre_syscap_info_path, "syscap.json")) 257 target_parts_list = _get_required_build_parts_list(target_platform_parts) 258 syscap_info_list = parts_config_info.get('syscap_info') 259 target_syscap_with_part_name_list = [] 260 target_syscap_list = [] 261 target_syscap_for_init_list = [] 262 all_syscap_list = [] 263 for syscap in syscap_info_list: 264 if syscap['component'] not in target_parts_list: 265 continue 266 if 'syscap' not in syscap or syscap['syscap'] == None or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 267 continue 268 for syscap_string in syscap['syscap']: 269 all_syscap_list.append(syscap_string.split('=')[0].strip()) 270 271 for key, value in syscap_product_dict['part_to_syscap'].items(): 272 for syscap in value: 273 if syscap not in all_syscap_list: 274 raise Exception( 275 "In config.json of part [{}],the syscap[{}] is incorrect, \ 276 please check the syscap name".format(key, syscap)) 277 278 for syscap in syscap_info_list: 279 remove_list = [] 280 if syscap['component'] not in target_parts_list: 281 continue 282 if 'syscap' not in syscap or syscap['syscap'] == None or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 283 continue 284 for syscap_string in syscap['syscap']: 285 if syscap_string.startswith("SystemCapability.") == True: 286 target_syscap_init_str = "const." 287 syscap_name = syscap_string.split('=')[0].strip() 288 all_syscap_product = syscap_product_dict['syscap'] 289 if syscap_name in all_syscap_product and not all_syscap_product[syscap_name]: 290 remove_list.append(syscap_string) 291 continue 292 elif syscap_name in all_syscap_product and all_syscap_product[syscap_name]: 293 target_syscap_init_str += syscap_name + '=true\n' 294 else: 295 if syscap_string.endswith('true'): 296 target_syscap_init_str += syscap_name + '=true\n' 297 elif syscap_string.endswith('false'): 298 remove_list.append(syscap_string) 299 continue 300 else: 301 target_syscap_init_str += syscap_string + "=true\n" 302 if target_syscap_init_str not in target_syscap_for_init_list: 303 target_syscap_for_init_list.append(target_syscap_init_str) 304 else: 305 raise Exception("""In bundle.json of part [{}], The syscap string [{}] is incorrect, 306 need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string)) 307 308 for remove_str in remove_list: 309 syscap['syscap'].remove(remove_str) 310 for i in range(len(syscap['syscap'])): 311 if syscap['syscap'][i].endswith('true') or syscap['syscap'][i].endswith('false'): 312 syscap['syscap'][i] = syscap['syscap'][i].split('=')[0].strip() 313 314 syscap['syscap'].sort() 315 target_syscap_with_part_name_list.append(syscap) 316 target_syscap_list.extend(syscap['syscap']) 317 318 # Generate SystemCapability.json & syscap.json & syscap.para 319 target_syscap_list.sort() 320 syscap_info_dict = read_json_file(os.path.join(pre_syscap_info_path, "SystemCapability.json")) 321 syscap_info_dict.update({'syscap':{'os':target_syscap_list}}) 322 system_etc_path = os.path.join(system_path, "etc/") 323 if not os.path.exists(system_path): 324 os.mkdir(system_path) 325 if not os.path.exists(system_etc_path): 326 os.mkdir(system_etc_path) 327 syscap_info_json = os.path.join(system_etc_path, "SystemCapability.json") 328 write_json_file(syscap_info_json, syscap_info_dict) 329 target_syscap_with_part_name_list.sort(key = syscap_sort) 330 syscap_info_with_part_name_file = os.path.join(system_etc_path, "syscap.json") 331 write_json_file(syscap_info_with_part_name_file, {'components': target_syscap_with_part_name_list}) 332 if not os.path.exists(os.path.join(system_etc_path, "param/")): 333 os.mkdir(os.path.join(system_etc_path, "param/")) 334 target_syscap_for_init_file = os.path.join(system_etc_path, "param/syscap.para") 335 f = open(target_syscap_for_init_file, "w") 336 f.writelines(target_syscap_for_init_list) 337 f.close() 338 339def load(args): 340 source_root_dir = args.source_root_dir 341 _check_args(args, source_root_dir) 342 config_output_relpath = os.path.join(args.gn_root_out_dir, 'build_configs') 343 344 # loading subsystem info, scan src dir and get subsystem ohos.build 345 _subsystem_info = subsystem_info.get_subsystem_info( 346 args.subsystem_config_file, args.example_subsystem_file, 347 source_root_dir, config_output_relpath, args.os_level) 348 349 target_arch = '{}_{}'.format(args.target_os, args.target_cpu) 350 # loading platforms config 351 _platforms_info = platforms_loader.get_platforms_info( 352 args.platforms_config_file, source_root_dir, args.gn_root_out_dir, 353 target_arch, config_output_relpath, args.scalable_build) 354 355 # get build platforms list 356 toolchain_to_variant_dict = _platforms_info.get('variant_toolchain_info') 357 variant_toolchains = toolchain_to_variant_dict.get('platform_toolchain') 358 _all_platforms = variant_toolchains.keys() 359 360 if args.build_platform_name == 'all': 361 build_platforms = _all_platforms 362 elif args.build_platform_name in _all_platforms: 363 build_platforms = [args.build_platform_name] 364 else: 365 raise Exception( 366 "The target_platform is incorrect, only allows [{}].".format( 367 ', '.join(_all_platforms))) 368 369 # loading ohos.build and gen part variant info 370 parts_config_info = load_ohos_build.get_parts_info( 371 source_root_dir, config_output_relpath, _subsystem_info, 372 variant_toolchains, target_arch, args.ignore_api_check, 373 args.exclusion_modules_config_file, args.load_test_config, 374 args.build_xts) 375 # check parts_config_info 376 _check_parts_config_info(parts_config_info) 377 parts_variants = parts_config_info.get('parts_variants') 378 parts_targets = parts_config_info.get('parts_targets') 379 parts_info = parts_config_info.get('parts_info') 380 381 config_output_dir = os.path.join(source_root_dir, config_output_relpath) 382 383 # target_platforms_parts.json 384 target_platform_parts = _get_platforms_all_parts( 385 source_root_dir, args.target_os, args.target_cpu, 386 _platforms_info.get('all_parts'), build_platforms, parts_variants) 387 target_platform_parts_file = os.path.join(config_output_dir, 388 "target_platforms_parts.json") 389 write_json_file(target_platform_parts_file, 390 target_platform_parts, 391 check_changes=True) 392 393 # {platform}_system_capabilities.json 394 # we assume that platform and devicetype are the same. 395 for platform in build_platforms: 396 platform_parts = target_platform_parts.get(platform) 397 platform_capabilities = [] 398 for _, origin in platform_parts.items(): 399 # parts_info.get() might be None if the part is a binary package 400 all_parts_variants = parts_info.get(origin) 401 if all_parts_variants is None: 402 continue 403 part = all_parts_variants[0] 404 if part.get('system_capabilities'): 405 entry = part.get('system_capabilities') 406 if len(entry) > 0: 407 platform_capabilities.extend(entry) 408 platform_part_json_file = os.path.join( 409 config_output_dir, "{0}_system_capabilities.json".format(platform)) 410 write_json_file(platform_part_json_file, 411 sorted(platform_capabilities), 412 check_changes=True) 413 414 target_platform_stubs = _get_platforms_all_stubs( 415 source_root_dir, args.target_os, args.target_cpu, 416 _platforms_info.get('all_stubs'), build_platforms, parts_variants) 417 generate_targets_gn.gen_stub_targets( 418 parts_config_info.get('parts_kits_info'), target_platform_stubs, 419 config_output_dir) 420 421 # platforms_parts_by_src.json 422 platforms_parts_by_src = _get_platforms_parts(parts_targets, 423 target_platform_parts) 424 platforms_parts_by_src_file = os.path.join(source_root_dir, 425 config_output_relpath, 426 "platforms_parts_by_src.json") 427 write_json_file(platforms_parts_by_src_file, 428 platforms_parts_by_src, 429 check_changes=True) 430 431 required_parts_targets = _get_required_build_targets( 432 parts_targets, target_platform_parts) 433 generate_targets_gn.gen_targets_gn(required_parts_targets, 434 config_output_dir) 435 _phony_target = parts_config_info.get('phony_target') 436 required_phony_targets = _get_required_build_targets( 437 _phony_target, target_platform_parts) 438 generate_targets_gn.gen_phony_targets(required_phony_targets, 439 config_output_dir) 440 441 # required_parts_targets.json 442 build_targets_info_file = os.path.join(config_output_dir, 443 "required_parts_targets.json") 444 write_json_file(build_targets_info_file, required_parts_targets) 445 # required_parts_targets_list.json 446 build_targets_list_file = os.path.join(config_output_dir, 447 "required_parts_targets_list.json") 448 write_json_file(build_targets_list_file, 449 list(required_parts_targets.values())) 450 451 # parts src flag file 452 parts_src_flag_file = os.path.join(config_output_dir, 453 "parts_src_flag.json") 454 write_json_file(parts_src_flag_file, 455 _get_parts_src_list(required_parts_targets, parts_info), 456 check_changes=True) 457 # write auto install part file 458 auto_install_list = _get_auto_install_list(parts_config_info.get("parts_path_info")) 459 auto_install_list_file = os.path.join(config_output_dir, "auto_install_parts.json") 460 write_json_file(auto_install_list_file, auto_install_list) 461 462 # write platforms_list.gni 463 platforms_list_gni_file = os.path.join(config_output_dir, 464 "platforms_list.gni") 465 _platforms = set(build_platforms) 466 _gni_file_content = [] 467 _gni_file_content.append('target_platform_list = [') 468 _gni_file_content.append(' "{}"'.format('",\n "'.join(_platforms))) 469 _gni_file_content.append(']') 470 _gni_file_content.append('kits_platform_list = [') 471 _gni_file_content.append(' "{}",'.format('",\n "'.join(_platforms))) 472 if 'phone' not in build_platforms: 473 _gni_file_content.append(' "phone"') 474 _gni_file_content.append(']') 475 write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) 476 477 # parts_different_info.json 478 # Generate parts differences in different platforms, using phone as base. 479 parts_different_info = _get_parts_by_platform(target_platform_parts) 480 parts_different_info_file = os.path.join(config_output_dir, 481 "parts_different_info.json") 482 write_json_file(parts_different_info_file, 483 parts_different_info, 484 check_changes=True) 485 # for testfwk 486 infos_for_testfwk_file = os.path.join(config_output_dir, 487 "infos_for_testfwk.json") 488 _output_infos_for_testfwk(parts_config_info, target_platform_parts, 489 infos_for_testfwk_file) 490 491 # check part feature 492 _check_product_part_feature(parts_info, 493 os.path.dirname(args.platforms_config_file)) 494 pre_syscap_info_path = os.path.dirname(args.platforms_config_file) 495 system_path = os.path.join(source_root_dir, os.path.join(os.path.dirname(args.platforms_config_file), "system/")) 496 generate_syscap_files(parts_config_info, target_platform_parts, pre_syscap_info_path, system_path) 497 498def _output_infos_by_platform(part_name_infos, parts_info_dict): 499 required_parts = {} 500 subsystem_infos = {} 501 for part_name, origin_part_name in part_name_infos.items(): 502 part_info = parts_info_dict.get(part_name) 503 if part_info is None: 504 continue 505 if origin_part_name != part_info.get('origin_part_name'): 506 raise Exception("part configuration is incorrect.") 507 required_parts[origin_part_name] = part_info 508 _subsystem_name = part_info.get('subsystem_name') 509 if _subsystem_name in subsystem_infos: 510 p_list = subsystem_infos.get(_subsystem_name) 511 else: 512 p_list = [] 513 p_list.append(origin_part_name) 514 subsystem_infos[_subsystem_name] = p_list 515 result = {} 516 result['subsystem_infos'] = subsystem_infos 517 result['part_infos'] = required_parts 518 return result 519 520def _output_infos_for_testfwk(parts_config_info, target_platform_parts, 521 infos_for_testfwk_file): 522 parts_info = parts_config_info.get('parts_info') 523 parts_info_dict = {} 524 for _part_name, _parts in parts_info.items(): 525 for _info in _parts: 526 parts_info_dict[_info.get('part_name')] = _info 527 528 _output_infos = {} 529 for _platform, _parts in target_platform_parts.items(): 530 result = _output_infos_by_platform(_parts, parts_info_dict) 531 _output_infos[_platform] = result 532 533 write_json_file(infos_for_testfwk_file, _output_infos, check_changes=True) 534 535 536def main(): 537 parser = argparse.ArgumentParser() 538 parser.add_argument('--platforms-config-file', required=True) 539 parser.add_argument('--subsystem-config-file', required=True) 540 parser.add_argument('--example-subsystem-file', required=False) 541 parser.add_argument('--exclusion-modules-config-file', required=False) 542 parser.add_argument('--source-root-dir', required=True) 543 parser.add_argument('--gn-root-out-dir', default='.') 544 parser.add_argument('--build-platform-name', default='phone') 545 parser.add_argument('--build-xts', dest='build_xts', action='store_true') 546 parser.set_defaults(build_xts=False) 547 parser.add_argument('--load-test-config', action='store_true') 548 parser.add_argument('--target-os', default='ohos') 549 parser.add_argument('--target-cpu', default='arm64') 550 parser.add_argument('--os-level', default='standard') 551 parser.add_argument('--ignore-api-check', nargs='*', default=[]) 552 553 parser.add_argument('--scalable-build', action='store_true') 554 parser.set_defaults(scalable_build=False) 555 args = parser.parse_args() 556 557 load(args) 558 return 0 559 560 561if __name__ == '__main__': 562 sys.exit(main()) 563