1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2023 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18import os 19 20from services.interface.load_interface import LoadInterface 21from containers.status import throw_exception 22from exceptions.ohos_exception import OHOSException 23from util.loader import platforms_loader # noqa: E402 24from util.loader import generate_targets_gn # noqa: E402 25from util.loader import load_ohos_build # noqa: E402 26from util.loader import subsystem_scan # noqa: E402 27from util.loader import subsystem_info # noqa: E402 28from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 29from util.log_util import LogUtil 30 31 32class OHOSLoader(LoadInterface): 33 34 def __init__(self): 35 super().__init__() 36 self.source_root_dir = "" 37 self.gn_root_out_dir = "" 38 self.os_level = "" 39 self.target_cpu = "" 40 self.target_os = "" 41 self.config_output_relpath = "" 42 self.config_output_dir = "" 43 self.target_arch = "" 44 self.subsystem_config_file = "" 45 self.subsystem_config_overlay_file = "" 46 self.platforms_config_file = "" 47 self.exclusion_modules_config_file = "" 48 self.example_subsystem_file = "" 49 self.build_example = "" 50 self.scalable_build = "" 51 self.build_platform_name = "" 52 self.build_xts = "" 53 self.ignore_api_check = "" 54 self.load_test_config = "" 55 self.subsystem_configs = "" 56 self._subsystem_info = "" 57 self.skip_partlist_check = "" 58 59 def __post_init__(self): 60 self.source_root_dir = self.config.root_path + '/' 61 self.gn_root_out_dir = self.config.out_path if not self.config.out_path.startswith( 62 '/') else os.path.relpath(self.config.out_path, self.config.root_path) 63 self.os_level = self.config.os_level if self.config.os_level else "standard" 64 self.target_cpu = self.config.target_cpu if self.config.target_cpu else "arm" 65 self.target_os = self.config.target_os if self.config.target_os else "ohos" 66 self.config_output_relpath = os.path.join( 67 self.gn_root_out_dir, 'build_configs') 68 self.config_output_dir = os.path.join( 69 self.source_root_dir, self.config_output_relpath) 70 self.target_arch = '{}_{}'.format(self.target_os, self.target_cpu) 71 self.subsystem_config_file = os.path.join( 72 self.config.root_path, 'out/preloader', self.config.product, 'subsystem_config.json') 73 self.platforms_config_file = os.path.join( 74 self.config.root_path, 'out/preloader', self.config.product, 'platforms.build') 75 self.exclusion_modules_config_file = os.path.join( 76 self.config.root_path, 'out/preloader', self.config.product, 'exclusion_modules.json') 77 self.example_subsystem_file = os.path.join( 78 self.config.root_path, 'build', 'subsystem_config_example.json') 79 80 compile_standard_allow_file = os.path.join( 81 self.config.root_path, 'out/preloader', self.config.product, 'compile_standard_whitelist.json') 82 compile_standard_allow_info = read_json_file(compile_standard_allow_file) 83 bundle_subsystem_allow_list = compile_standard_allow_info.get("bundle_subsystem_error", []) 84 85 # check config args 86 self._check_args() 87 88 self.build_example = self.args_dict.get('build_example') 89 if not self.build_example: 90 self.example_subsystem_file = "" 91 self.scalable_build = self.args_dict.get('scalable_build') 92 self.build_platform_name = self.args_dict.get('build_platform_name') 93 self.build_xts = self.args_dict.get('build_xts') 94 self.ignore_api_check = self.args_dict.get('ignore_api_check') 95 self.load_test_config = self.args_dict.get('load_test_config') 96 self.skip_partlist_check = self.args_dict.get('skip_partlist_check') 97 98 self._subsystem_info = subsystem_info.get_subsystem_info( 99 self.subsystem_config_file, 100 self.example_subsystem_file, 101 self.source_root_dir, 102 self.config_output_relpath, 103 self.os_level) 104 overrided_components = self._override_components() 105 106 self._platforms_info = platforms_loader.get_platforms_info( 107 self.platforms_config_file, 108 self.source_root_dir, 109 self.gn_root_out_dir, 110 self.target_arch, 111 self.config_output_relpath, 112 self.scalable_build) 113 self.variant_toolchains = self._platforms_info.get( 114 'variant_toolchain_info').get('platform_toolchain') 115 self._all_platforms = self.variant_toolchains.keys() 116 self.build_platforms = self._get_build_platforms() 117 self.parts_config_info = load_ohos_build.get_parts_info( 118 self.source_root_dir, 119 self.config_output_relpath, 120 self._subsystem_info, 121 self.variant_toolchains, 122 self.target_arch, 123 self.ignore_api_check, 124 self.exclusion_modules_config_file, 125 self.load_test_config, 126 overrided_components, 127 bundle_subsystem_allow_list, 128 self.skip_partlist_check, 129 self.build_xts) 130 self.parts_targets = self.parts_config_info.get('parts_targets') 131 self.phony_targets = self.parts_config_info.get('phony_target') 132 self.parts_info = self.parts_config_info.get('parts_info') 133 self.target_platform_parts = self._get_platforms_all_parts() 134 self.target_platform_stubs = self._get_platforms_all_stubs() 135 self.required_parts_targets_list = self._get_required_build_parts_list() 136 self.required_phony_targets = self._get_required_phony_targets() 137 self.required_parts_targets = self._get_required_build_targets() 138 139# check method 140 141 '''Description: Check the parameters passed in config. If the parameters are not 142 specified or the file content pointed to by the parameters does not 143 exist, an exception will be thrown directly. 144 @parameter:none 145 @return :none 146 ''' 147 @throw_exception 148 def _check_args(self): 149 LogUtil.hb_info("Checking all build args...") 150 # check subsystem_config_file 151 if not read_json_file(self.subsystem_config_file): 152 self.subsystem_config_file = os.path.join( 153 self.source_root_dir, 'build/subsystem_config.json') 154 if not read_json_file(self.subsystem_config_file): 155 raise OHOSException("Cannot get the content from platform config file, \ 156 please check whether the corresponding file('out/preloader/{}/subsystem_config.json' or \ 157 'build/subsystem_config.json') is written correctly.".format(self.config.product), "2001") 158 159 # check gn_root_out_dir 160 if not self.gn_root_out_dir: 161 raise OHOSException("Args gn_root_out_dir is required.", "2002") 162 if not os.path.realpath(self.gn_root_out_dir).startswith(self.source_root_dir): 163 raise OHOSException("Args gn_root_out_dir is incorrect.", "2003") 164 165 # check platform config file 166 if not read_json_file(self.platforms_config_file): 167 raise OHOSException("Cannot get the content from platform config file, \ 168 please check whether the corresponding file('out/preloader/${product_name}/platforms.build') \ 169 is written correctly.".format(self.config.product), "2004") 170 171 # check example subsystem file 172 if not read_json_file(self.example_subsystem_file): 173 raise OHOSException("Cannot get the content from example subsystem file, please check whether \ 174 the corresponding file ('build/subsystem_config_example.json') exists.", "2005") 175 176 @throw_exception 177 def _check_product_part_feature(self): 178 LogUtil.hb_info("Checking all product features...") 179 product_preloader_dir = os.path.dirname(self.platforms_config_file) 180 _preloader_feature_file = os.path.join(product_preloader_dir, 181 'features.json') 182 _preloader_feature_info = read_json_file(_preloader_feature_file) 183 part_to_feature = _preloader_feature_info.get('part_to_feature') 184 for key, vals in part_to_feature.items(): 185 part = self.parts_info.get(key) 186 if part is None: 187 continue 188 _p_info = part[0] 189 def_feature_list = _p_info.get('feature_list') 190 if not def_feature_list: 191 continue 192 for _f_name in vals: 193 if _f_name not in def_feature_list: 194 raise OHOSException( 195 "The product use a feature that is not supported" 196 " by this part, part_name='{}', feature='{}'".format( 197 key, _f_name), "2006") 198 199 @throw_exception 200 def _check_parts_config_info(self): 201 LogUtil.hb_info("Checking parts config...") 202 if not ('parts_info' in self.parts_config_info 203 and 'subsystem_parts' in self.parts_config_info 204 and 'parts_variants' in self.parts_config_info 205 and 'parts_kits_info' in self.parts_config_info 206 and 'parts_inner_kits_info' in self.parts_config_info 207 and 'parts_targets' in self.parts_config_info): 208 raise OHOSException( 209 "Loading ohos.build information is incorrect.", "2007") 210 211# generate method 212 213 '''Description: Generate SystemCapability.json & syscap.json & syscap.para, dir:[ 214 (//out/preloader/${product_name}/system/etc/SystemCapability.json), 215 (//out/preloader/${product_name}/system/etc/syscap.json), 216 (//out/preloader/${product_name}/system/etc/param/syscap.para)] 217 @parameter:none 218 @return :none 219 ''' 220 @throw_exception 221 def _generate_syscap_files(self): 222 pre_syscap_info_path = os.path.dirname(self.platforms_config_file) 223 system_path = os.path.join(self.source_root_dir, os.path.join( 224 os.path.dirname(self.platforms_config_file), "system/")) 225 syscap_product_dict = read_json_file( 226 os.path.join(pre_syscap_info_path, "syscap.json")) 227 syscap_info_list = self.parts_config_info.get('syscap_info') 228 target_syscap_with_part_name_list = [] 229 target_syscap_list = [] 230 target_syscap_for_init_list = [] 231 all_syscap_list = [] 232 for syscap in syscap_info_list: 233 if syscap['component'] not in self.required_parts_targets_list: 234 continue 235 if 'syscap' not in syscap or syscap['syscap'] is None \ 236 or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 237 continue 238 for syscap_string in syscap['syscap']: 239 all_syscap_list.append(syscap_string.split('=')[0].strip()) 240 241 for key, value in syscap_product_dict['part_to_syscap'].items(): 242 for syscap in value: 243 if syscap not in all_syscap_list: 244 raise OHOSException( 245 "In config.json of part [{}],the syscap[{}] is incorrect, \ 246 please check the syscap name".format(key, syscap), "2008") 247 248 for syscap in syscap_info_list: 249 remove_list = [] 250 if syscap['component'] not in self.required_parts_targets_list: 251 continue 252 if 'syscap' not in syscap or syscap['syscap'] is None \ 253 or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 254 continue 255 for syscap_string in syscap['syscap']: 256 if syscap_string.startswith("SystemCapability.") is True: 257 target_syscap_init_str = "const." 258 syscap_name = syscap_string.split('=')[0].strip() 259 all_syscap_product = syscap_product_dict['syscap'] 260 if syscap_name in all_syscap_product and not all_syscap_product[syscap_name]: 261 remove_list.append(syscap_string) 262 continue 263 elif syscap_name in all_syscap_product and all_syscap_product[syscap_name]: 264 target_syscap_init_str += syscap_name + '=true\n' 265 else: 266 if syscap_string.endswith('true'): 267 target_syscap_init_str += syscap_name + '=true\n' 268 elif syscap_string.endswith('false'): 269 remove_list.append(syscap_string) 270 continue 271 else: 272 target_syscap_init_str += syscap_string + "=true\n" 273 if target_syscap_init_str not in target_syscap_for_init_list: 274 target_syscap_for_init_list.append( 275 target_syscap_init_str) 276 else: 277 raise OHOSException("""In bundle.json of part [{}], The syscap string [{}] is incorrect, 278 need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string), "2009") 279 280 for remove_str in remove_list: 281 syscap['syscap'].remove(remove_str) 282 for i in range(len(syscap['syscap'])): 283 if syscap['syscap'][i].endswith('true') or syscap['syscap'][i].endswith('false'): 284 syscap['syscap'][i] = syscap['syscap'][i].split('=')[ 285 0].strip() 286 287 syscap['syscap'].sort() 288 target_syscap_with_part_name_list.append(syscap) 289 target_syscap_list.extend(syscap['syscap']) 290 291 # Generate SystemCapability.json & syscap.json & syscap.para 292 target_syscap_list.sort() 293 syscap_info_dict = read_json_file(os.path.join( 294 pre_syscap_info_path, "SystemCapability.json")) 295 syscap_info_dict.update({'syscap': {'os': target_syscap_list}}) 296 system_etc_path = os.path.join(system_path, "etc/") 297 if not os.path.exists(system_path): 298 os.mkdir(system_path) 299 if not os.path.exists(system_etc_path): 300 os.mkdir(system_etc_path) 301 syscap_info_json = os.path.join( 302 system_etc_path, "SystemCapability.json") 303 write_json_file(syscap_info_json, syscap_info_dict) 304 LogUtil.hb_info( 305 "generate syscap info file to '{}'".format(syscap_info_json), mode=self.config.log_mode) 306 target_syscap_with_part_name_list.sort( 307 key=lambda syscap: syscap['component']) 308 syscap_info_with_part_name_file = os.path.join( 309 system_etc_path, "syscap.json") 310 write_json_file(syscap_info_with_part_name_file, { 311 'components': target_syscap_with_part_name_list}) 312 LogUtil.hb_info("generate syscap info with part name list to '{}'".format( 313 syscap_info_with_part_name_file), mode=self.config.log_mode) 314 if not os.path.exists(os.path.join(system_etc_path, "param/")): 315 os.mkdir(os.path.join(system_etc_path, "param/")) 316 target_syscap_for_init_file = os.path.join( 317 system_etc_path, "param/syscap.para") 318 with open(target_syscap_for_init_file, "w") as file: 319 file.writelines(sorted(target_syscap_for_init_list)) 320 LogUtil.hb_info("generate target syscap for init list to '{}'".format( 321 target_syscap_for_init_file), mode=self.config.log_mode) 322 323 '''Description: output infos for testfwk into a json file. \ 324 (/out/${product_name}/build_configs/infos_for_testfwk.json) 325 @parameter:none 326 @return :none 327 ''' 328 329 def _generate_infos_for_testfwk(self): 330 infos_for_testfwk_file = os.path.join(self.config_output_dir, 331 "infos_for_testfwk.json") 332 parts_info = self.parts_config_info.get('parts_info') 333 parts_info_dict = {} 334 for _part_name, _parts in parts_info.items(): 335 for _info in _parts: 336 parts_info_dict[_info.get('part_name')] = _info 337 _output_infos = {} 338 for _platform, _parts in self.target_platform_parts.items(): 339 result = self._output_infos_by_platform(_parts, parts_info_dict) 340 _output_infos[_platform] = result 341 write_json_file(infos_for_testfwk_file, 342 _output_infos, check_changes=True) 343 LogUtil.hb_info("generate infos for testfwk to '{}'".format( 344 infos_for_testfwk_file), mode=self.config.log_mode) 345 346 '''Description: output all target platform parts into a json file \ 347 (/out/${product_name}/build_configs/target_platforms_parts.json) 348 @parameter:none 349 @return :none 350 ''' 351 352 def _generate_target_platform_parts(self): 353 target_platform_parts_file = os.path.join(self.config_output_dir, 354 "target_platforms_parts.json") 355 write_json_file(target_platform_parts_file, 356 self.target_platform_parts, 357 check_changes=True) 358 LogUtil.hb_info("generate target platform parts to '{}'".format( 359 target_platform_parts_file), mode=self.config.log_mode) 360 361 '''Description: Generate parts differences in different platforms, using phone as base. \ 362 (/out/${product_name}/build_configs/parts_different_info.json) 363 @parameter: none 364 @return :none 365 ''' 366 367 def _generate_part_different_info(self): 368 parts_different_info = self._get_parts_by_platform() 369 parts_different_info_file = os.path.join(self.config_output_dir, 370 "parts_different_info.json") 371 write_json_file(parts_different_info_file, 372 parts_different_info, 373 check_changes=True) 374 LogUtil.hb_info("generate part different info to '{}'".format( 375 parts_different_info_file), mode=self.config.log_mode) 376 377 '''Description: output platforms list into a gni file. \ 378 (/out/${product_name}/build_configs/platforms_list.gni) 379 @parameter: none 380 @return: none 381 ''' 382 383 def _generate_platforms_list(self): 384 platforms_list_gni_file = os.path.join(self.config_output_dir, 385 "platforms_list.gni") 386 _platforms = set(self.build_platforms) 387 _gni_file_content = ['target_platform_list = [', ' "{}"'.format('",\n "'.join(_platforms)), ']', 388 'kits_platform_list = [', ' "{}",'.format('",\n "'.join(_platforms))] 389 if 'phone' not in self.build_platforms: 390 _gni_file_content.append(' "phone"') 391 _gni_file_content.append(']') 392 write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) 393 LogUtil.hb_info("generate platforms list to '{}'".format( 394 platforms_list_gni_file), mode=self.config.log_mode) 395 396 '''Description: output auto install part into a json file. \ 397 (/out/${product_name}/build_configs/auto_install_parts.json) 398 @parameter: none 399 @return: none 400 ''' 401 402 def _generate_auto_install_part(self): 403 parts_path_info = self.parts_config_info.get("parts_path_info") 404 auto_install_part_list = [] 405 for part, path in parts_path_info.items(): 406 if str(path).startswith("drivers/interface") or \ 407 str(path).startswith("third_party"): 408 auto_install_part_list.append(part) 409 auto_install_list_file = os.path.join( 410 self.config_output_dir, "auto_install_parts.json") 411 write_json_file(auto_install_list_file, auto_install_part_list) 412 LogUtil.hb_info("generate auto install part to '{}'".format( 413 auto_install_list_file), mode=self.config.log_mode) 414 415 '''Description: output src flag into a json file. \ 416 (/out/${product_name}/build_configs/parts_src_flag.json) 417 @parameter: none 418 @return :none 419 ''' 420 421 def _generate_src_flag(self): 422 parts_src_flag_file = os.path.join(self.config_output_dir, 423 "parts_src_flag.json") 424 write_json_file(parts_src_flag_file, 425 self._get_parts_src_list(), 426 check_changes=True) 427 LogUtil.hb_info( 428 "generated parts src flag to '{}/subsystem_info/parts_src_flag.json'".format( 429 self.config_output_dir), mode=self.config.log_mode) 430 431 '''Description: output build target list into a json file.\ 432 (/out/${product_name}/build_configs/required_parts_targets_list.json) 433 @parameter: none 434 @return :none 435 ''' 436 437 def _generate_required_parts_targets_list(self): 438 build_targets_list_file = os.path.join(self.config_output_dir, 439 "required_parts_targets_list.json") 440 write_json_file(build_targets_list_file, 441 list(self.required_parts_targets.values())) 442 LogUtil.hb_info("generate build targets list file to '{}'".format( 443 build_targets_list_file), mode=self.config.log_mode) 444 445 '''Description: output build target info into a json file. \ 446 (/out/${product_name}/build_configs/required_parts_targets.json) 447 @parameter: none 448 @return: none 449 ''' 450 451 def _generate_required_parts_targets(self): 452 build_targets_info_file = os.path.join(self.config_output_dir, 453 "required_parts_targets.json") 454 write_json_file(build_targets_info_file, self.required_parts_targets) 455 LogUtil.hb_info("generate required parts targets to '{}'".format( 456 build_targets_info_file), mode=self.config.log_mode) 457 458 '''Description: output platforms part by src into a json file. \ 459 (/out/${product_name}/build_configs/platforms_parts_by_src.json) 460 @parameter: none 461 @return :none 462 ''' 463 464 def _generate_platforms_part_by_src(self): 465 platforms_parts_by_src = self._get_platforms_parts() 466 platforms_parts_by_src_file = os.path.join(self.source_root_dir, 467 self.config_output_relpath, 468 "platforms_parts_by_src.json") 469 write_json_file(platforms_parts_by_src_file, 470 platforms_parts_by_src, 471 check_changes=True) 472 LogUtil.hb_info("generated platforms parts by src to '{}'".format( 473 platforms_parts_by_src_file), mode=self.config.log_mode) 474 475 '''Description: output system configs info into 4 files:[ 476 (/out/${product_name}/build_configs/subsystem_info/parts_list.gni), 477 (/out/${product_name}/build_configs/subsystem_info/inner_kits_list.gni), 478 (/out/${product_name}/build_configs/subsystem_info/system_kits_list.gni), 479 (/out/${product_name}/build_configs/subsystem_info/parts_test_list.gni), 480 (/out/${product_name}/build_configs/subsystem_info/BUILD.gn)] 481 @parameter: none 482 @return :none 483 ''' 484 485 def _generate_target_gn(self): 486 generate_targets_gn.gen_targets_gn(self.required_parts_targets, 487 self.config_output_dir) 488 489 '''Description: output phony targets build file. \ 490 (/out/${product_name}/build_configs/phony_target/BUILD.gn) 491 @parameter: none 492 @return :none 493 ''' 494 495 def _generate_phony_targets_build_file(self): 496 generate_targets_gn.gen_phony_targets(self.required_phony_targets, 497 self.config_output_dir) 498 499 '''Description: output system configs info into 2 files:[ 500 (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/BUILG.gn), 501 (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/zframework_stub_exists.gni)] 502 @parameter: none 503 @return :none 504 ''' 505 506 def _generate_stub_targets(self): 507 generate_targets_gn.gen_stub_targets( 508 self.parts_config_info.get('parts_kits_info'), 509 self.target_platform_stubs, 510 self.config_output_dir) 511 512 '''Description: output system capabilities into a json file. \ 513 (/out/${product_name}/build_configs/${platform}_system_capabilities.json) 514 @parameter: none 515 @return :none 516 ''' 517 518 def _generate_system_capabilities(self): 519 for platform in self.build_platforms: 520 platform_parts = self.target_platform_parts.get(platform) 521 platform_capabilities = [] 522 for _, origin in platform_parts.items(): 523 # parts_info.get() might be None if the part is a binary package 524 all_parts_variants = self.parts_info.get(origin) 525 if all_parts_variants is None: 526 continue 527 part = all_parts_variants[0] 528 if part.get('system_capabilities'): 529 entry = part.get('system_capabilities') 530 if len(entry) > 0: 531 platform_capabilities.extend(entry) 532 platform_part_json_file = os.path.join( 533 self.config_output_dir, "{0}_system_capabilities.json".format(platform)) 534 write_json_file(platform_part_json_file, 535 sorted(platform_capabilities), 536 check_changes=True) 537 LogUtil.hb_info( 538 "generated system capabilities to '{}/{}_system_capabilities.json'".format( 539 self.config_output_dir, platform), mode=self.config.log_mode) 540 541 '''Description: output system configs info into three json files:[ 542 (/out/${product_name}/build_configs/subsystem_info/subsystem_build_config.json), 543 (/out/${product_name}/build_configs/subsystem_info/src_subsystem_info.json), 544 (/out/${product_name}/build_configs/subsystem_info/no_src_subsystem_info.json)] 545 @parameter: none 546 @return :none 547 ''' 548 549 def _generate_subsystem_configs(self): 550 551 # The function has been implemented in module util/loader/subsystem_info.py 552 LogUtil.hb_info( 553 "generated subsystem build config to '{}/subsystem_info/subsystem_build_config.json'".format( 554 self.config_output_dir), mode=self.config.log_mode) 555 LogUtil.hb_info( 556 "generated src subsystem info to '{}/subsystem_info/src_subsystem_info.json'".format( 557 self.config_output_dir), mode=self.config.log_mode) 558 LogUtil.hb_info( 559 "generated no src subsystem info to '{}/subsystem_info/no_src_subsystem_info.json'".format( 560 self.config_output_dir), mode=self.config.log_mode) 561 562# get method 563 @throw_exception 564 def _get_build_platforms(self) -> list: 565 build_platforms = [] 566 if self.build_platform_name == 'all': 567 build_platforms = self._all_platforms 568 elif self.build_platform_name in self._all_platforms: 569 build_platforms = [self.build_platform_name] 570 else: 571 raise OHOSException( 572 "The target_platform is incorrect, only allows [{}].".format( 573 ', '.join(self._all_platforms)), "2010") 574 return build_platforms 575 576 def _get_parts_by_platform(self) -> dict: 577 parts_info = {} 578 if 'phone' in self.target_platform_parts: 579 phone_parts_list = self.target_platform_parts.get('phone').keys() 580 else: 581 phone_parts_list = [] 582 for _platform, _parts_info in self.target_platform_parts.items(): 583 base_parts_list = [] 584 curr_parts_list = [] 585 for _real_name, _original_name in _parts_info.items(): 586 if _real_name in phone_parts_list: 587 base_parts_list.append(_real_name) 588 elif _original_name in phone_parts_list: 589 base_parts_list.append(_real_name) 590 else: 591 curr_parts_list.append(_real_name) 592 result_data = { 593 "base_parts_list": base_parts_list, 594 "curr_parts_list": curr_parts_list 595 } 596 parts_info[_platform] = result_data 597 return parts_info 598 599 def _get_platforms_all_parts(self) -> dict: 600 _dist_parts_variants = self._load_component_dist() 601 target_platform_parts = {} 602 all_parts = self._platforms_info.get('all_parts') 603 parts_variants = self.parts_config_info.get('parts_variants') 604 for _platform, _parts in all_parts.items(): 605 if _platform not in self.build_platforms: 606 continue 607 part_name_info = {} 608 for part_def in _parts: 609 real_name, original_name = self._get_real_part_name( 610 part_def, _platform, parts_variants) 611 if real_name is None: 612 # find this from component_dist 613 real_name, original_name = self._get_real_part_name( 614 part_def, _platform, _dist_parts_variants) 615 if real_name is None: 616 continue 617 part_name_info[real_name] = original_name 618 target_platform_parts[_platform] = part_name_info 619 return target_platform_parts 620 621 def _get_platforms_all_stubs(self) -> dict: 622 _dist_parts_variants = self._load_component_dist() 623 platform_stubs = {} 624 all_stubs = self._platforms_info.get('all_stubs') 625 parts_variants = self.parts_config_info.get('parts_variants') 626 for _platform, _part_names in all_stubs.items(): 627 if _platform not in self.build_platforms: 628 continue 629 stub_parts_from_src = [] 630 stub_parts_from_dist = [] 631 for part_name in _part_names: 632 real_name, original_name = self._get_real_part_name( 633 part_name, _platform, parts_variants) 634 # real_name=None means part_name doesn't exist in source tree, 635 # use binary in component_dist then. 636 if real_name is None: 637 # find this from component_dist 638 real_name, original_name = self._get_real_part_name( 639 part_name, _platform, _dist_parts_variants) 640 if real_name is None: 641 continue 642 else: 643 stub_sources = os.path.join( 644 self.source_root_dir, 645 "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt" # noqa: E501 646 .format(self.target_os, self.target_cpu, real_name)) 647 stub_parts_from_dist.append( 648 '"{}"'.format(stub_sources)) 649 else: 650 stub_parts_from_src.append(real_name) 651 platform_stubs[_platform] = { 652 "src": stub_parts_from_src, 653 "dist": stub_parts_from_dist, 654 } 655 return platform_stubs 656 657 def _get_platforms_parts(self) -> dict: 658 platforms_parts = {} 659 src_parts_targets = self.parts_targets 660 src_all_parts = src_parts_targets.keys() 661 for _platform, _all_parts in self.target_platform_parts.items(): 662 src_parts_list = [] 663 no_src_parts_list = [] 664 for _part in _all_parts.keys(): 665 if _part in src_all_parts: 666 src_parts_list.append(_part) 667 else: 668 no_src_parts_list.append(_part) 669 _data = { 670 'src_parts': src_parts_list, 671 'no_src_parts': no_src_parts_list 672 } 673 platforms_parts[_platform] = _data 674 return platforms_parts 675 676 def _get_parts_src_list(self) -> list: 677 parts_name_map = {} 678 for _list in self.parts_info.values(): 679 for _info in _list: 680 parts_name_map[_info.get('part_name')] = _info.get( 681 'origin_part_name') 682 _src_set = set() 683 for _name in self.required_parts_targets.keys(): 684 _origin_name = parts_name_map.get(_name) 685 if _origin_name is None: 686 continue 687 _src_set.add(_origin_name) 688 return list(_src_set) 689 690 def _get_required_build_targets(self) -> dict: 691 required_build_targets = {} 692 for _p_name, _info in self.parts_targets.items(): 693 if _p_name not in self.required_parts_targets_list: 694 continue 695 required_build_targets[_p_name] = _info 696 return required_build_targets 697 698 def _get_required_phony_targets(self) -> dict: 699 required_build_targets = {} 700 for _p_name, _info in self.phony_targets.items(): 701 if _p_name not in self.required_parts_targets_list: 702 continue 703 required_build_targets[_p_name] = _info 704 return required_build_targets 705 706 def _get_required_build_parts_list(self) -> list: 707 parts_set = set() 708 for _parts_list in self.target_platform_parts.values(): 709 parts_set.update(_parts_list) 710 return list(parts_set) 711 712# util method 713 714 def _load_component_dist(self) -> dict: 715 _parts_variants_info = {} 716 _dir = "component_dist/{}-{}/packages_to_install".format( 717 self.target_os, self.target_cpu) 718 _file_name = "dist_parts_info.json" 719 _dist_parts_info_file = os.path.join( 720 self.source_root_dir, _dir, _file_name) 721 if not os.path.exists(_dist_parts_info_file): 722 # If the file does not exist, do nothing and return 723 return _parts_variants_info 724 _parts_info = read_json_file(_dist_parts_info_file) 725 if _parts_info is None: 726 raise Exception("read file '{}' failed.".format( 727 _dist_parts_info_file)) 728 for _part_info in _parts_info: 729 origin_part_name = _part_info.get('origin_part_name') 730 if origin_part_name in _parts_variants_info: 731 variants = _parts_variants_info.get(origin_part_name) 732 else: 733 variants = [] 734 _variant_name = _part_info.get('variant_name') 735 variants.append(_variant_name) 736 _parts_variants_info[origin_part_name] = variants 737 return _parts_variants_info 738 739 def _get_real_part_name(self, original_part_name: str, current_platform: str, parts_variants: dict): 740 part_info = parts_variants.get(original_part_name) 741 if part_info is None: 742 return None, None 743 if current_platform in part_info and current_platform != 'phone': 744 real_name = '{}_{}'.format(original_part_name, current_platform) 745 else: 746 real_name = original_part_name 747 return real_name, original_part_name 748 749 '''Description: called by _out_infos_for_testfwk, output information by platform 750 @parameter:none 751 @return :none 752 ''' 753 754 def _output_infos_by_platform(self, part_name_infos: dict, parts_info_dict: dict): 755 required_parts = {} 756 subsystem_infos = {} 757 for part_name, origin_part_name in part_name_infos.items(): 758 part_info = parts_info_dict.get(part_name) 759 if part_info is None: 760 continue 761 if origin_part_name != part_info.get('origin_part_name'): 762 raise Exception("part configuration is incorrect.") 763 required_parts[origin_part_name] = part_info 764 _subsystem_name = part_info.get('subsystem_name') 765 if _subsystem_name in subsystem_infos: 766 p_list = subsystem_infos.get(_subsystem_name) 767 else: 768 p_list = [] 769 p_list.append(origin_part_name) 770 subsystem_infos[_subsystem_name] = p_list 771 result = {} 772 result['subsystem_infos'] = subsystem_infos 773 result['part_infos'] = required_parts 774 return result 775 776 def _execute_loader_args_display(self): 777 LogUtil.hb_info('Loading configuration file...') 778 args = [] 779 args.append('platforms_config_file="{}"'.format( 780 self.platforms_config_file)) 781 args.append('subsystem_config_file="{}"'.format( 782 self.subsystem_config_file)) 783 args.append('example_subsystem_file="{}"'.format( 784 self.example_subsystem_file)) 785 args.append('exclusion_modules_config_file="{}"'.format( 786 self.exclusion_modules_config_file)) 787 args.append('source_root_dir="{}"'.format(self.source_root_dir)) 788 args.append('gn_root_out_dir="{}"'.format(self.gn_root_out_dir)) 789 args.append('build_platform_name={}'.format(self.build_platform_name)) 790 args.append('build_xts={}'.format(self.build_xts)) 791 args.append('load_test_config={}'.format(self.load_test_config)) 792 args.append('target_os={}'.format(self.target_os)) 793 args.append('target_cpu={}'.format(self.target_cpu)) 794 args.append('os_level={}'.format(self.os_level)) 795 args.append('ignore_api_check={}'.format(self.ignore_api_check)) 796 args.append('scalable_build={}'.format(self.scalable_build)) 797 args.append('skip_partlist_check={}'.format(self.skip_partlist_check)) 798 LogUtil.write_log(self.config.log_path, 799 'loader args:{}'.format(args), 'info') 800 801 def _override_components(self): 802 '''Description: Check whether there are components that need to be replaced, and if so, 803 replace the component configuration file bundle.json in subsystem_info and update 804 the component list generated by the preloader. 805 @parameter:none 806 @return :overrided_components 807 ''' 808 parts_file = self.platforms_config_file.replace( 809 "platforms.build", "parts.json") 810 all_parts = read_json_file(parts_file) 811 if "parts" not in all_parts: 812 LogUtil.hb_warning("{} does not contain parts!".format(parts_file)) 813 return {} 814 overrided = False 815 overrided_components = {} 816 all_parts = all_parts["parts"] 817 component_override_map = {} 818 all_component_override_map = {} 819 for subsystem_name, build_config_info in self._subsystem_info.items(): 820 if "build_files" not in build_config_info: 821 continue 822 823 # scan all bundle.json or ohos.build files with named groups 824 for build_file in build_config_info["build_files"]: 825 826 # ohos.build does not support overrided components 827 if not build_file.endswith('bundle.json'): 828 continue 829 830 # Only device or vendor components can do named groups extensions 831 if (not build_file.startswith(self.source_root_dir + 'device/')) \ 832 and (not build_file.startswith(self.source_root_dir + 'vendor/')): 833 continue 834 835 # "subsystem", "name" and "override" is required 836 component = read_json_file(build_file).get("component") 837 838 if (not component) or (not all(key in component for key in ("subsystem", "name", "override"))): 839 continue 840 841 full_part_name = f"{component.get('subsystem')}:{component.get('name')}" 842 if full_part_name not in all_parts: 843 LogUtil.hb_warning("{} was not configured for this product: {}".format( 844 build_file, full_part_name)) 845 continue 846 847 if self._override_one_component(self._subsystem_info, component, build_file, all_parts, overrided_components, component_override_map): 848 overrided = True 849 850 if overrided: 851 # Update parts.json and parts_config.json generated by preloader 852 write_json_file(parts_file, {"parts": all_parts}) 853 parts_file = self.platforms_config_file.replace( 854 "platforms.build", "parts_config.json") 855 self._output_parts_config_json(all_parts, parts_file) 856 all_component_override_map.update(component_override_map) 857 write_json_file( 858 f"{self.config_output_dir}/component_override_map.json", all_component_override_map) 859 return overrided_components 860 861 def _override_one_component(self, subsystem_info: dict, component: dict, build_file: str, all_parts: dict, overrided_components: dict, component_override_map: dict): 862 '''Description: Perform a replacement of a single component and return the component list update result. 863 @parameter:subsystem_info, component, build_file, all_parts, overrided_components 864 @return :True or False(Whether replacement has been performed) 865 ''' 866 splits = component["override"].split(":") 867 if len(splits) != 2: 868 LogUtil.hb_warning( 869 "{} override value is invalid format. Skip override process".format(build_file)) 870 return False 871 overrided_subsystem = splits[0] 872 overrided_component = splits[1] 873 if overrided_subsystem not in subsystem_info: 874 LogUtil.hb_warning( 875 "{} override invalid subsystem. Skip override process".format(build_file)) 876 return False 877 878 founded_bundle = "" 879 880 for bundle in subsystem_info[overrided_subsystem]["build_files"]: 881 if not bundle.endswith('bundle.json'): 882 continue 883 884 bundle_obj = read_json_file(bundle) 885 886 if bundle_obj.get("component", {}).get("name") == overrided_component: 887 founded_bundle = bundle 888 break 889 890 if founded_bundle: 891 origin_component = read_json_file(build_file).get('component') 892 LogUtil.hb_warning( 893 f"You are trying to override \"{component['override']}\" with \"{origin_component.get('subsystem')}:{origin_component.get('name')}\". \nPlease ensure that the modules in \"{component['override']}\" only rely on the interfaces of other components through \"external_deps\"") 894 895 # replace bundle.json in subsystem_info's build_files 896 subsystem_info[overrided_subsystem]["build_files"].remove( 897 founded_bundle) 898 899 # Update parts.json generated by preloader, which means that new added components will not be installed 900 # Ensure that the overrided components will be installed 901 full_partname = f"{overrided_subsystem}:{overrided_component}" 902 if full_partname in all_parts: 903 all_parts.remove(full_partname) 904 905 overrided_components[f"{component['subsystem']}:{component['name']}"] = { 906 'subsystem': overrided_subsystem, 907 'partName': overrided_component 908 } 909 component_override_map[overrided_component] = component["name"] 910 return True 911 LogUtil.hb_warning( 912 "{}:{} is not configured in product, \new add component will be installed!".format( 913 overrided_subsystem, overrided_component)) 914 return False 915 916 def _output_parts_config_json(self, all_parts: dict, output_file: dict): 917 '''Description: Update the parts list file generated by preloader 918 @parameter: all_parts, output_file 919 @return :none 920 ''' 921 parts_config = {} 922 for part in all_parts: 923 part = part.replace(":", "_") 924 part = part.replace("-", "_") 925 part = part.replace(".", "_") 926 part = part.replace("/", "_") 927 parts_config[part] = True 928 write_json_file(output_file, parts_config) 929