1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2023 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18import os 19import json 20import copy 21 22from services.interface.load_interface import LoadInterface 23from containers.status import throw_exception 24from exceptions.ohos_exception import OHOSException 25from util.loader import platforms_loader # noqa: E402 26from util.loader import generate_targets_gn # noqa: E402 27from util.loader import load_ohos_build # noqa: E402 28from util.loader import subsystem_scan # noqa: E402 29from util.loader import subsystem_info # noqa: E402 30from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 31from util.log_util import LogUtil 32from resources.config import Config 33 34 35class OHOSLoader(LoadInterface): 36 37 def __init__(self): 38 super().__init__() 39 self.source_root_dir = "" 40 self.gn_root_out_dir = "" 41 self.os_level = "" 42 self.target_cpu = "" 43 self.target_os = "" 44 self.config_output_relpath = "" 45 self.config_output_dir = "" 46 self.target_arch = "" 47 self.subsystem_config_file = "" 48 self.subsystem_config_overlay_file = "" 49 self.platforms_config_file = "" 50 self.exclusion_modules_config_file = "" 51 self.example_subsystem_file = "" 52 self.build_example = "" 53 self.scalable_build = "" 54 self.build_platform_name = "" 55 self.build_xts = "" 56 self.ignore_api_check = "" 57 self.load_test_config = "" 58 self.subsystem_configs = "" 59 self._subsystem_info = "" 60 self.skip_partlist_check = "" 61 62 def __post_init__(self): 63 self.source_root_dir = self.config.root_path + '/' 64 self.gn_root_out_dir = self.config.out_path if not self.config.out_path.startswith( 65 '/') else os.path.relpath(self.config.out_path, self.config.root_path) 66 self.os_level = self.config.os_level if self.config.os_level else "standard" 67 self.target_cpu = self.config.target_cpu if self.config.target_cpu else "arm" 68 self.target_os = self.config.target_os if self.config.target_os else "ohos" 69 self.config_output_relpath = os.path.join( 70 self.gn_root_out_dir, 'build_configs') 71 self.config_output_dir = os.path.join( 72 self.source_root_dir, self.config_output_relpath) 73 self.target_arch = '{}_{}'.format(self.target_os, self.target_cpu) 74 self.subsystem_config_file = os.path.join( 75 self.config.root_path, 'out/preloader', self.config.product, 'subsystem_config.json') 76 self.platforms_config_file = os.path.join( 77 self.config.root_path, 'out/preloader', self.config.product, 'platforms.build') 78 self.exclusion_modules_config_file = os.path.join( 79 self.config.root_path, 'out/preloader', self.config.product, 'exclusion_modules.json') 80 self.example_subsystem_file = os.path.join( 81 self.config.root_path, 'build', 'subsystem_config_example.json') 82 self.parts_src_file = os.path.join( 83 self.config_output_dir, 'parts_src_flag.json') 84 self.auto_install_file = os.path.join( 85 self.config_output_dir, 'auto_install_parts.json') 86 self.components_file = os.path.join( 87 self.config_output_dir, 'parts_info', 'components.json') 88 89 compile_standard_allow_file = os.path.join( 90 self.config.root_path, 'out/preloader', self.config.product, 'compile_standard_whitelist.json') 91 compile_standard_allow_info = read_json_file(compile_standard_allow_file) 92 bundle_subsystem_allow_list = compile_standard_allow_info.get("bundle_subsystem_error", []) 93 94 # check config args 95 self._check_args() 96 97 self.build_example = self.args_dict.get('build_example') 98 if not self.build_example: 99 self.example_subsystem_file = "" 100 self.scalable_build = self.args_dict.get('scalable_build') 101 self.build_platform_name = self.args_dict.get('build_platform_name') 102 self.build_xts = self.args_dict.get('build_xts') 103 self.ignore_api_check = self.args_dict.get('ignore_api_check') 104 self.load_test_config = self.args_dict.get('load_test_config') 105 self.skip_partlist_check = self.args_dict.get('skip_partlist_check') 106 107 self._subsystem_info = subsystem_info.get_subsystem_info( 108 self.subsystem_config_file, 109 self.example_subsystem_file, 110 self.source_root_dir, 111 self.config_output_relpath, 112 self.os_level) 113 overrided_components = self._override_components() 114 115 self._platforms_info = platforms_loader.get_platforms_info( 116 self.platforms_config_file, 117 self.source_root_dir, 118 self.gn_root_out_dir, 119 self.target_arch, 120 self.config_output_relpath, 121 self.scalable_build) 122 self.variant_toolchains = self._platforms_info.get( 123 'variant_toolchain_info').get('platform_toolchain') 124 self._all_platforms = self.variant_toolchains.keys() 125 self.build_platforms = self._get_build_platforms() 126 self.parts_config_info = load_ohos_build.get_parts_info( 127 self.source_root_dir, 128 self.config_output_relpath, 129 self._subsystem_info, 130 self.variant_toolchains, 131 self.target_arch, 132 self.ignore_api_check, 133 self.exclusion_modules_config_file, 134 self.load_test_config, 135 overrided_components, 136 bundle_subsystem_allow_list, 137 self.skip_partlist_check, 138 self.build_xts) 139 self.parts_targets = self.parts_config_info.get('parts_targets') 140 self.phony_targets = self.parts_config_info.get('phony_target') 141 self.parts_info = self.parts_config_info.get('parts_info') 142 self.target_platform_parts = self._get_platforms_all_parts() 143 self.target_platform_stubs = self._get_platforms_all_stubs() 144 self.required_parts_targets_list = self._get_required_build_parts_list() 145 self.required_phony_targets = self._get_required_phony_targets() 146 self.required_parts_targets = self._get_required_build_targets() 147 148 149 @throw_exception 150 def _merge_components_info(self, components): 151 config = Config() 152 sdk_components_file = os.path.join(config.root_path, "out/products_ext/components.json") 153 if not os.path.exists(sdk_components_file): 154 return 155 156 sdk_components_info = read_json_file(sdk_components_file) 157 for name, val in sdk_components_info.items(): 158 if name not in components.keys(): 159 components[name] = val 160 161 162 @throw_exception 163 def _cropping_components(self): 164 src_parts = read_json_file(self.parts_src_file) 165 166 auto_parts = read_json_file(self.auto_install_file) 167 168 self.third_party_file = os.path.join(self.config.root_path, "out/products_ext/third_party_allow_list.json") 169 if not os.path.exists(self.third_party_file): 170 self.third_party_file = os.path.join(self.config.root_path, 'build/third_party_allow_list.json') 171 cropping_parts = read_json_file(self.third_party_file) 172 173 # auto_install_whitelist 174 self.auto_install_file = os.path.join(self.config.root_path, "out/products_ext/auto_install_whitelist.json") 175 if not os.path.exists(self.auto_install_file): 176 self.auto_install_file = os.path.join(self.config.root_path, 'build/auto_install_whitelist.json') 177 add_parts = read_json_file(self.auto_install_file) 178 179 components_data = read_json_file(self.components_file) 180 181 new_components_data = copy.deepcopy(components_data) 182 183 for component, component_value in components_data.items(): 184 if component not in src_parts and component not in auto_parts and component not in cropping_parts \ 185 and component not in add_parts: 186 del new_components_data[component] 187 self._merge_components_info(new_components_data) 188 os.remove(self.components_file) 189 write_json_file(self.components_file, new_components_data) 190 191 192# check method 193 194 '''Description: Check the parameters passed in config. If the parameters are not 195 specified or the file content pointed to by the parameters does not 196 exist, an exception will be thrown directly. 197 @parameter:none 198 @return :none 199 ''' 200 @throw_exception 201 def _check_args(self): 202 LogUtil.hb_info("Checking all build args...") 203 # check subsystem_config_file 204 if not read_json_file(self.subsystem_config_file): 205 self.subsystem_config_file = os.path.join( 206 self.source_root_dir, 'build/subsystem_config.json') 207 if not read_json_file(self.subsystem_config_file): 208 raise OHOSException("Cannot get the content from platform config file, \ 209 please check whether the corresponding file('out/preloader/{}/subsystem_config.json' or \ 210 'build/subsystem_config.json') is written correctly.".format(self.config.product), "2001") 211 212 # check gn_root_out_dir 213 if not self.gn_root_out_dir: 214 raise OHOSException("Args gn_root_out_dir is required.", "2002") 215 if not os.path.realpath(self.gn_root_out_dir).startswith(self.source_root_dir): 216 raise OHOSException("Args gn_root_out_dir is incorrect.", "2003") 217 218 # check platform config file 219 if not read_json_file(self.platforms_config_file): 220 raise OHOSException("Cannot get the content from platform config file, \ 221 please check whether the corresponding file('out/preloader/${product_name}/platforms.build') \ 222 is written correctly.".format(self.config.product), "2004") 223 224 # check example subsystem file 225 if not read_json_file(self.example_subsystem_file): 226 raise OHOSException("Cannot get the content from example subsystem file, please check whether \ 227 the corresponding file ('build/subsystem_config_example.json') exists.", "2005") 228 229 @throw_exception 230 def _check_product_part_feature(self): 231 LogUtil.hb_info("Checking all product features...") 232 product_preloader_dir = os.path.dirname(self.platforms_config_file) 233 _preloader_feature_file = os.path.join(product_preloader_dir, 234 'features.json') 235 _preloader_feature_info = read_json_file(_preloader_feature_file) 236 part_to_feature = _preloader_feature_info.get('part_to_feature') 237 _feature_whitelist_file = os.path.join( 238 self.source_root_dir, "out/products_ext", "component_feature_whitelist.json" 239 ) 240 if not os.path.exists(_feature_whitelist_file): 241 _feature_whitelist_file = os.path.join( 242 self.source_root_dir, "build/", "component_feature_whitelist.json" 243 ) 244 _feature_whitelist_info = read_json_file(_feature_whitelist_file) 245 _feature_whitelist_list = [] 246 if _feature_whitelist_info: 247 _feature_whitelist_list = list(_feature_whitelist_info.keys()) 248 for key, vals in part_to_feature.items(): 249 part = self.parts_info.get(key) 250 if part is None: 251 continue 252 _p_info = part[0] 253 def_feature_list = _p_info.get('feature_list') 254 if vals and not def_feature_list: 255 message = "The product use a feature vals='{}', but that is not defined " \ 256 "in this part bundle.json file, part_name='{}'".format(vals, key) 257 if key not in _feature_whitelist_list: 258 raise OHOSException(message, "2006") 259 LogUtil.hb_warning(message) 260 continue 261 for _f_name in vals: 262 if _f_name not in def_feature_list: 263 raise OHOSException( 264 "The product use a feature that is not supported" 265 " by this part, part_name='{}', feature='{}'".format( 266 key, _f_name), "2006") 267 268 @throw_exception 269 def _check_parts_config_info(self): 270 LogUtil.hb_info("Checking parts config...") 271 if not ('parts_info' in self.parts_config_info 272 and 'subsystem_parts' in self.parts_config_info 273 and 'parts_variants' in self.parts_config_info 274 and 'parts_kits_info' in self.parts_config_info 275 and 'parts_inner_kits_info' in self.parts_config_info 276 and 'parts_targets' in self.parts_config_info): 277 raise OHOSException( 278 "Loading ohos.build information is incorrect.", "2007") 279 280# generate method 281 282 '''Description: Generate SystemCapability.json & syscap.json & syscap.para, dir:[ 283 (//out/preloader/${product_name}/system/etc/SystemCapability.json), 284 (//out/preloader/${product_name}/system/etc/syscap.json), 285 (//out/preloader/${product_name}/system/etc/param/syscap.para)] 286 @parameter:none 287 @return :none 288 ''' 289 @throw_exception 290 def _generate_syscap_files(self): 291 pre_syscap_info_path = os.path.dirname(self.platforms_config_file) 292 system_path = os.path.join(self.source_root_dir, os.path.join( 293 os.path.dirname(self.platforms_config_file), "system/")) 294 syscap_product_dict = read_json_file( 295 os.path.join(pre_syscap_info_path, "syscap.json")) 296 syscap_info_list = self.parts_config_info.get('syscap_info') 297 target_syscap_with_part_name_list = [] 298 target_syscap_list = [] 299 target_syscap_for_init_list = [] 300 all_syscap_list = [] 301 for syscap in syscap_info_list: 302 if syscap['component'] not in self.required_parts_targets_list: 303 continue 304 if 'syscap' not in syscap or syscap['syscap'] is None \ 305 or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 306 continue 307 for syscap_string in syscap['syscap']: 308 all_syscap_list.append(syscap_string.split('=')[0].strip()) 309 310 for key, value in syscap_product_dict['part_to_syscap'].items(): 311 part = self.parts_info.get(key) 312 if part is None: 313 continue 314 for syscap in value: 315 if syscap not in all_syscap_list: 316 raise OHOSException( 317 "In config.json of part [{}],the syscap[{}] is incorrect, \ 318 please check the syscap name".format(key, syscap), "2008") 319 320 for syscap in syscap_info_list: 321 remove_list = [] 322 if syscap['component'] not in self.required_parts_targets_list: 323 continue 324 if 'syscap' not in syscap or syscap['syscap'] is None \ 325 or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 326 continue 327 for syscap_string in syscap['syscap']: 328 if syscap_string.startswith("SystemCapability.") is True: 329 target_syscap_init_str = "const." 330 syscap_name = syscap_string.split('=')[0].strip() 331 all_syscap_product = syscap_product_dict['syscap'] 332 if syscap_name in all_syscap_product and not all_syscap_product[syscap_name]: 333 remove_list.append(syscap_string) 334 continue 335 elif syscap_name in all_syscap_product and all_syscap_product[syscap_name]: 336 target_syscap_init_str += syscap_name + '=true\n' 337 else: 338 if syscap_string.endswith('true'): 339 target_syscap_init_str += syscap_name + '=true\n' 340 elif syscap_string.endswith('false'): 341 remove_list.append(syscap_string) 342 continue 343 else: 344 target_syscap_init_str += syscap_string + "=true\n" 345 if target_syscap_init_str not in target_syscap_for_init_list: 346 target_syscap_for_init_list.append( 347 target_syscap_init_str) 348 else: 349 raise OHOSException("""In bundle.json of part [{}], The syscap string [{}] is incorrect, 350 need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string), "2009") 351 352 for remove_str in remove_list: 353 syscap['syscap'].remove(remove_str) 354 for i in range(len(syscap['syscap'])): 355 if syscap['syscap'][i].endswith('true') or syscap['syscap'][i].endswith('false'): 356 syscap['syscap'][i] = syscap['syscap'][i].split('=')[ 357 0].strip() 358 359 syscap['syscap'].sort() 360 target_syscap_with_part_name_list.append(syscap) 361 target_syscap_list.extend(syscap['syscap']) 362 363 # Generate SystemCapability.json & syscap.json & syscap.para 364 target_syscap_list.sort() 365 syscap_info_dict = read_json_file(os.path.join( 366 pre_syscap_info_path, "SystemCapability.json")) 367 syscap_info_dict.update({'syscap': {'os': target_syscap_list}}) 368 system_etc_path = os.path.join(system_path, "etc/") 369 if not os.path.exists(system_path): 370 os.mkdir(system_path) 371 if not os.path.exists(system_etc_path): 372 os.mkdir(system_etc_path) 373 syscap_info_json = os.path.join( 374 system_etc_path, "SystemCapability.json") 375 write_json_file(syscap_info_json, syscap_info_dict) 376 LogUtil.hb_info( 377 "generate syscap info file to '{}'".format(syscap_info_json), mode=self.config.log_mode) 378 target_syscap_with_part_name_list.sort( 379 key=lambda syscap: syscap['component']) 380 syscap_info_with_part_name_file = os.path.join( 381 system_etc_path, "syscap.json") 382 write_json_file(syscap_info_with_part_name_file, { 383 'components': target_syscap_with_part_name_list}) 384 LogUtil.hb_info("generate syscap info with part name list to '{}'".format( 385 syscap_info_with_part_name_file), mode=self.config.log_mode) 386 if not os.path.exists(os.path.join(system_etc_path, "param/")): 387 os.mkdir(os.path.join(system_etc_path, "param/")) 388 target_syscap_for_init_file = os.path.join( 389 system_etc_path, "param/syscap.para") 390 with open(target_syscap_for_init_file, "w") as file: 391 file.writelines(sorted(target_syscap_for_init_list)) 392 LogUtil.hb_info("generate target syscap for init list to '{}'".format( 393 target_syscap_for_init_file), mode=self.config.log_mode) 394 395# get method 396 @throw_exception 397 def _get_build_platforms(self) -> list: 398 build_platforms = [] 399 if self.build_platform_name == 'all': 400 build_platforms = self._all_platforms 401 elif self.build_platform_name in self._all_platforms: 402 build_platforms = [self.build_platform_name] 403 else: 404 raise OHOSException( 405 "The target_platform is incorrect, only allows [{}].".format( 406 ', '.join(self._all_platforms)), "2010") 407 return build_platforms 408 409 '''Description: output infos for testfwk into a json file. \ 410 (/out/${product_name}/build_configs/infos_for_testfwk.json) 411 @parameter:none 412 @return :none 413 ''' 414 415 def _generate_infos_for_testfwk(self): 416 infos_for_testfwk_file = os.path.join(self.config_output_dir, 417 "infos_for_testfwk.json") 418 parts_info = self.parts_config_info.get('parts_info') 419 parts_info_dict = {} 420 for _part_name, _parts in parts_info.items(): 421 for _info in _parts: 422 parts_info_dict[_info.get('part_name')] = _info 423 _output_infos = {} 424 for _platform, _parts in self.target_platform_parts.items(): 425 result = self._output_infos_by_platform(_parts, parts_info_dict) 426 _output_infos[_platform] = result 427 write_json_file(infos_for_testfwk_file, 428 _output_infos, check_changes=True) 429 LogUtil.hb_info("generate infos for testfwk to '{}'".format( 430 infos_for_testfwk_file), mode=self.config.log_mode) 431 432 '''Description: output all target platform parts into a json file \ 433 (/out/${product_name}/build_configs/target_platforms_parts.json) 434 @parameter:none 435 @return :none 436 ''' 437 438 def _generate_target_platform_parts(self): 439 target_platform_parts_file = os.path.join(self.config_output_dir, 440 "target_platforms_parts.json") 441 write_json_file(target_platform_parts_file, 442 self.target_platform_parts, 443 check_changes=True) 444 LogUtil.hb_info("generate target platform parts to '{}'".format( 445 target_platform_parts_file), mode=self.config.log_mode) 446 447 '''Description: Generate parts differences in different platforms, using phone as base. \ 448 (/out/${product_name}/build_configs/parts_different_info.json) 449 @parameter: none 450 @return :none 451 ''' 452 453 def _generate_part_different_info(self): 454 parts_different_info = self._get_parts_by_platform() 455 parts_different_info_file = os.path.join(self.config_output_dir, 456 "parts_different_info.json") 457 write_json_file(parts_different_info_file, 458 parts_different_info, 459 check_changes=True) 460 LogUtil.hb_info("generate part different info to '{}'".format( 461 parts_different_info_file), mode=self.config.log_mode) 462 463 '''Description: output platforms list into a gni file. \ 464 (/out/${product_name}/build_configs/platforms_list.gni) 465 @parameter: none 466 @return: none 467 ''' 468 469 def _generate_platforms_list(self): 470 platforms_list_gni_file = os.path.join(self.config_output_dir, 471 "platforms_list.gni") 472 _platforms = set(self.build_platforms) 473 _gni_file_content = ['target_platform_list = [', ' "{}"'.format('",\n "'.join(_platforms)), ']', 474 'kits_platform_list = [', ' "{}",'.format('",\n "'.join(_platforms))] 475 if 'phone' not in self.build_platforms: 476 _gni_file_content.append(' "phone"') 477 _gni_file_content.append(']') 478 write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) 479 LogUtil.hb_info("generate platforms list to '{}'".format( 480 platforms_list_gni_file), mode=self.config.log_mode) 481 482 '''Description: output auto install part into a json file. \ 483 (/out/${product_name}/build_configs/auto_install_parts.json) 484 @parameter: none 485 @return: none 486 ''' 487 488 def _generate_auto_install_part(self): 489 parts_path_info = self.parts_config_info.get("parts_path_info") 490 auto_install_part_list = [] 491 auto_install_list_file = os.path.join( 492 self.config_output_dir, "auto_install_parts.json") 493 write_json_file(auto_install_list_file, auto_install_part_list) 494 LogUtil.hb_info("generate auto install part to '{}'".format( 495 auto_install_list_file), mode=self.config.log_mode) 496 497 '''Description: output src flag into a json file. \ 498 (/out/${product_name}/build_configs/parts_src_flag.json) 499 @parameter: none 500 @return :none 501 ''' 502 503 def _generate_src_flag(self): 504 parts_src_flag_file = os.path.join(self.config_output_dir, 505 "parts_src_flag.json") 506 write_json_file(parts_src_flag_file, 507 self._get_parts_src_list(), 508 check_changes=True) 509 LogUtil.hb_info( 510 "generated parts src flag to '{}/subsystem_info/parts_src_flag.json'".format( 511 self.config_output_dir), mode=self.config.log_mode) 512 513 '''Description: output build target list into a json file.\ 514 (/out/${product_name}/build_configs/required_parts_targets_list.json) 515 @parameter: none 516 @return :none 517 ''' 518 519 def _generate_required_parts_targets_list(self): 520 build_targets_list_file = os.path.join(self.config_output_dir, 521 "required_parts_targets_list.json") 522 write_json_file(build_targets_list_file, 523 list(self.required_parts_targets.values())) 524 LogUtil.hb_info("generate build targets list file to '{}'".format( 525 build_targets_list_file), mode=self.config.log_mode) 526 527 '''Description: output build target info into a json file. \ 528 (/out/${product_name}/build_configs/required_parts_targets.json) 529 @parameter: none 530 @return: none 531 ''' 532 533 def _generate_required_parts_targets(self): 534 build_targets_info_file = os.path.join(self.config_output_dir, 535 "required_parts_targets.json") 536 write_json_file(build_targets_info_file, self.required_parts_targets) 537 LogUtil.hb_info("generate required parts targets to '{}'".format( 538 build_targets_info_file), mode=self.config.log_mode) 539 540 '''Description: output platforms part by src into a json file. \ 541 (/out/${product_name}/build_configs/platforms_parts_by_src.json) 542 @parameter: none 543 @return :none 544 ''' 545 546 def _generate_platforms_part_by_src(self): 547 platforms_parts_by_src = self._get_platforms_parts() 548 platforms_parts_by_src_file = os.path.join(self.source_root_dir, 549 self.config_output_relpath, 550 "platforms_parts_by_src.json") 551 write_json_file(platforms_parts_by_src_file, 552 platforms_parts_by_src, 553 check_changes=True) 554 LogUtil.hb_info("generated platforms parts by src to '{}'".format( 555 platforms_parts_by_src_file), mode=self.config.log_mode) 556 557 '''Description: output system configs info into 4 files:[ 558 (/out/${product_name}/build_configs/subsystem_info/parts_list.gni), 559 (/out/${product_name}/build_configs/subsystem_info/inner_kits_list.gni), 560 (/out/${product_name}/build_configs/subsystem_info/system_kits_list.gni), 561 (/out/${product_name}/build_configs/subsystem_info/parts_test_list.gni), 562 (/out/${product_name}/build_configs/subsystem_info/BUILD.gn)] 563 @parameter: none 564 @return :none 565 ''' 566 567 def _generate_target_gn(self): 568 generate_targets_gn.gen_targets_gn(self.required_parts_targets, 569 self.config_output_dir) 570 571 '''Description: output phony targets build file. \ 572 (/out/${product_name}/build_configs/phony_target/BUILD.gn) 573 @parameter: none 574 @return :none 575 ''' 576 577 def _generate_phony_targets_build_file(self): 578 generate_targets_gn.gen_phony_targets(self.required_phony_targets, 579 self.config_output_dir) 580 581 '''Description: output system configs info into 2 files:[ 582 (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/BUILG.gn), 583 (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/zframework_stub_exists.gni)] 584 @parameter: none 585 @return :none 586 ''' 587 588 def _generate_stub_targets(self): 589 generate_targets_gn.gen_stub_targets( 590 self.parts_config_info.get('parts_kits_info'), 591 self.target_platform_stubs, 592 self.config_output_dir) 593 594 '''Description: output system capabilities into a json file. \ 595 (/out/${product_name}/build_configs/${platform}_system_capabilities.json) 596 @parameter: none 597 @return :none 598 ''' 599 600 def _generate_system_capabilities(self): 601 for platform in self.build_platforms: 602 platform_parts = self.target_platform_parts.get(platform) 603 platform_capabilities = [] 604 for _, origin in platform_parts.items(): 605 # parts_info.get() might be None if the part is a binary package 606 all_parts_variants = self.parts_info.get(origin) 607 if all_parts_variants is None: 608 continue 609 part = all_parts_variants[0] 610 if part.get('system_capabilities'): 611 entry = part.get('system_capabilities') 612 if len(entry) > 0: 613 platform_capabilities.extend(entry) 614 platform_part_json_file = os.path.join( 615 self.config_output_dir, "{0}_system_capabilities.json".format(platform)) 616 write_json_file(platform_part_json_file, 617 sorted(platform_capabilities), 618 check_changes=True) 619 LogUtil.hb_info( 620 "generated system capabilities to '{}/{}_system_capabilities.json'".format( 621 self.config_output_dir, platform), mode=self.config.log_mode) 622 623 '''Description: output system configs info into three json files:[ 624 (/out/${product_name}/build_configs/subsystem_info/subsystem_build_config.json), 625 (/out/${product_name}/build_configs/subsystem_info/src_subsystem_info.json), 626 (/out/${product_name}/build_configs/subsystem_info/no_src_subsystem_info.json)] 627 @parameter: none 628 @return :none 629 ''' 630 631 def _generate_subsystem_configs(self): 632 633 # The function has been implemented in module util/loader/subsystem_info.py 634 LogUtil.hb_info( 635 "generated subsystem build config to '{}/subsystem_info/subsystem_build_config.json'".format( 636 self.config_output_dir), mode=self.config.log_mode) 637 LogUtil.hb_info( 638 "generated src subsystem info to '{}/subsystem_info/src_subsystem_info.json'".format( 639 self.config_output_dir), mode=self.config.log_mode) 640 LogUtil.hb_info( 641 "generated no src subsystem info to '{}/subsystem_info/no_src_subsystem_info.json'".format( 642 self.config_output_dir), mode=self.config.log_mode) 643 644 def _get_parts_by_platform(self) -> dict: 645 parts_info = {} 646 if 'phone' in self.target_platform_parts: 647 phone_parts_list = self.target_platform_parts.get('phone').keys() 648 else: 649 phone_parts_list = [] 650 for _platform, _parts_info in self.target_platform_parts.items(): 651 base_parts_list = [] 652 curr_parts_list = [] 653 for _real_name, _original_name in _parts_info.items(): 654 if _real_name in phone_parts_list: 655 base_parts_list.append(_real_name) 656 elif _original_name in phone_parts_list: 657 base_parts_list.append(_real_name) 658 else: 659 curr_parts_list.append(_real_name) 660 result_data = { 661 "base_parts_list": base_parts_list, 662 "curr_parts_list": curr_parts_list 663 } 664 parts_info[_platform] = result_data 665 return parts_info 666 667 def _get_platforms_all_parts(self) -> dict: 668 _dist_parts_variants = self._load_component_dist() 669 target_platform_parts = {} 670 all_parts = self._platforms_info.get('all_parts') 671 parts_variants = self.parts_config_info.get('parts_variants') 672 for _platform, _parts in all_parts.items(): 673 if _platform not in self.build_platforms: 674 continue 675 part_name_info = {} 676 for part_def in _parts: 677 real_name, original_name = self._get_real_part_name( 678 part_def, _platform, parts_variants) 679 if real_name is None: 680 # find this from component_dist 681 real_name, original_name = self._get_real_part_name( 682 part_def, _platform, _dist_parts_variants) 683 if real_name is None: 684 continue 685 part_name_info[real_name] = original_name 686 target_platform_parts[_platform] = part_name_info 687 return target_platform_parts 688 689 def _get_platforms_all_stubs(self) -> dict: 690 _dist_parts_variants = self._load_component_dist() 691 platform_stubs = {} 692 all_stubs = self._platforms_info.get('all_stubs') 693 parts_variants = self.parts_config_info.get('parts_variants') 694 for _platform, _part_names in all_stubs.items(): 695 if _platform not in self.build_platforms: 696 continue 697 stub_parts_from_src = [] 698 stub_parts_from_dist = [] 699 for part_name in _part_names: 700 real_name, original_name = self._get_real_part_name( 701 part_name, _platform, parts_variants) 702 # real_name=None means part_name doesn't exist in source tree, 703 # use binary in component_dist then. 704 if real_name is None: 705 # find this from component_dist 706 real_name, original_name = self._get_real_part_name( 707 part_name, _platform, _dist_parts_variants) 708 if real_name is None: 709 continue 710 else: 711 stub_sources = os.path.join( 712 self.source_root_dir, 713 "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt" # noqa: E501 714 .format(self.target_os, self.target_cpu, real_name)) 715 stub_parts_from_dist.append( 716 '"{}"'.format(stub_sources)) 717 else: 718 stub_parts_from_src.append(real_name) 719 platform_stubs[_platform] = { 720 "src": stub_parts_from_src, 721 "dist": stub_parts_from_dist, 722 } 723 return platform_stubs 724 725 def _get_platforms_parts(self) -> dict: 726 platforms_parts = {} 727 src_parts_targets = self.parts_targets 728 src_all_parts = src_parts_targets.keys() 729 for _platform, _all_parts in self.target_platform_parts.items(): 730 src_parts_list = [] 731 no_src_parts_list = [] 732 for _part in _all_parts.keys(): 733 if _part in src_all_parts: 734 src_parts_list.append(_part) 735 else: 736 no_src_parts_list.append(_part) 737 _data = { 738 'src_parts': src_parts_list, 739 'no_src_parts': no_src_parts_list 740 } 741 platforms_parts[_platform] = _data 742 return platforms_parts 743 744 def _get_parts_src_list(self) -> list: 745 parts_name_map = {} 746 for _list in self.parts_info.values(): 747 for _info in _list: 748 parts_name_map[_info.get('part_name')] = _info.get( 749 'origin_part_name') 750 _src_set = set() 751 for _name in self.required_parts_targets.keys(): 752 _origin_name = parts_name_map.get(_name) 753 if _origin_name is None: 754 continue 755 _src_set.add(_origin_name) 756 return list(_src_set) 757 758 def _get_required_build_targets(self) -> dict: 759 required_build_targets = {} 760 for _p_name, _info in self.parts_targets.items(): 761 if _p_name not in self.required_parts_targets_list: 762 continue 763 required_build_targets[_p_name] = _info 764 return required_build_targets 765 766 def _get_required_phony_targets(self) -> dict: 767 required_build_targets = {} 768 for _p_name, _info in self.phony_targets.items(): 769 if _p_name not in self.required_parts_targets_list: 770 continue 771 required_build_targets[_p_name] = _info 772 return required_build_targets 773 774 def _get_required_build_parts_list(self) -> list: 775 parts_set = set() 776 for _parts_list in self.target_platform_parts.values(): 777 parts_set.update(_parts_list) 778 return list(parts_set) 779 780# util method 781 782 def _load_component_dist(self) -> dict: 783 _parts_variants_info = {} 784 _dir = "component_dist/{}-{}/packages_to_install".format( 785 self.target_os, self.target_cpu) 786 _file_name = "dist_parts_info.json" 787 _dist_parts_info_file = os.path.join( 788 self.source_root_dir, _dir, _file_name) 789 if not os.path.exists(_dist_parts_info_file): 790 # If the file does not exist, do nothing and return 791 return _parts_variants_info 792 _parts_info = read_json_file(_dist_parts_info_file) 793 if _parts_info is None: 794 raise Exception("read file '{}' failed.".format( 795 _dist_parts_info_file)) 796 for _part_info in _parts_info: 797 origin_part_name = _part_info.get('origin_part_name') 798 if origin_part_name in _parts_variants_info: 799 variants = _parts_variants_info.get(origin_part_name) 800 else: 801 variants = [] 802 _variant_name = _part_info.get('variant_name') 803 variants.append(_variant_name) 804 _parts_variants_info[origin_part_name] = variants 805 return _parts_variants_info 806 807 def _get_real_part_name(self, original_part_name: str, current_platform: str, parts_variants: dict): 808 part_info = parts_variants.get(original_part_name) 809 if part_info is None: 810 return None, None 811 if current_platform in part_info and current_platform != 'phone': 812 real_name = '{}_{}'.format(original_part_name, current_platform) 813 else: 814 real_name = original_part_name 815 return real_name, original_part_name 816 817 '''Description: called by _out_infos_for_testfwk, output information by platform 818 @parameter:none 819 @return :none 820 ''' 821 822 def _output_infos_by_platform(self, part_name_infos: dict, parts_info_dict: dict): 823 required_parts = {} 824 subsystem_infos = {} 825 for part_name, origin_part_name in part_name_infos.items(): 826 part_info = parts_info_dict.get(part_name) 827 if part_info is None: 828 continue 829 if origin_part_name != part_info.get('origin_part_name'): 830 raise Exception("part configuration is incorrect.") 831 required_parts[origin_part_name] = part_info 832 _subsystem_name = part_info.get('subsystem_name') 833 if _subsystem_name in subsystem_infos: 834 p_list = subsystem_infos.get(_subsystem_name) 835 else: 836 p_list = [] 837 p_list.append(origin_part_name) 838 subsystem_infos[_subsystem_name] = p_list 839 result = {} 840 result['subsystem_infos'] = subsystem_infos 841 result['part_infos'] = required_parts 842 return result 843 844 def _execute_loader_args_display(self): 845 LogUtil.hb_info('Loading configuration file...') 846 args = [] 847 args.append('platforms_config_file="{}"'.format( 848 self.platforms_config_file)) 849 args.append('subsystem_config_file="{}"'.format( 850 self.subsystem_config_file)) 851 args.append('example_subsystem_file="{}"'.format( 852 self.example_subsystem_file)) 853 args.append('exclusion_modules_config_file="{}"'.format( 854 self.exclusion_modules_config_file)) 855 args.append('source_root_dir="{}"'.format(self.source_root_dir)) 856 args.append('gn_root_out_dir="{}"'.format(self.gn_root_out_dir)) 857 args.append('build_platform_name={}'.format(self.build_platform_name)) 858 args.append('build_xts={}'.format(self.build_xts)) 859 args.append('load_test_config={}'.format(self.load_test_config)) 860 args.append('target_os={}'.format(self.target_os)) 861 args.append('target_cpu={}'.format(self.target_cpu)) 862 args.append('os_level={}'.format(self.os_level)) 863 args.append('ignore_api_check={}'.format(self.ignore_api_check)) 864 args.append('scalable_build={}'.format(self.scalable_build)) 865 args.append('skip_partlist_check={}'.format(self.skip_partlist_check)) 866 LogUtil.write_log(self.config.log_path, 867 'loader args:{}'.format(args), 'info') 868 869 def _override_components(self): 870 '''Description: Check whether there are components that need to be replaced, and if so, 871 replace the component configuration file bundle.json in subsystem_info and update 872 the component list generated by the preloader. 873 @parameter:none 874 @return :overrided_components 875 ''' 876 parts_file = self.platforms_config_file.replace( 877 "platforms.build", "parts.json") 878 all_parts = read_json_file(parts_file) 879 if "parts" not in all_parts: 880 LogUtil.hb_warning("{} does not contain parts!".format(parts_file)) 881 return {} 882 overrided = False 883 overrided_components = {} 884 all_parts = all_parts["parts"] 885 component_override_map = {} 886 all_component_override_map = {} 887 for subsystem_name, build_config_info in self._subsystem_info.items(): 888 if "build_files" not in build_config_info: 889 continue 890 891 # scan all bundle.json or ohos.build files with named groups 892 for build_file in build_config_info["build_files"]: 893 894 # ohos.build does not support overrided components 895 if not build_file.endswith('bundle.json'): 896 continue 897 898 # Only device or vendor components can do named groups extensions 899 if (not build_file.startswith(self.source_root_dir + 'device/')) \ 900 and (not build_file.startswith(self.source_root_dir + 'vendor/')): 901 continue 902 903 # "subsystem", "name" and "override" is required 904 component = read_json_file(build_file).get("component") 905 906 if (not component) or (not all(key in component for key in ("subsystem", "name", "override"))): 907 continue 908 909 full_part_name = f"{component.get('subsystem')}:{component.get('name')}" 910 if full_part_name not in all_parts: 911 LogUtil.hb_warning("{} was not configured for this product: {}".format( 912 build_file, full_part_name)) 913 continue 914 915 if self._override_one_component(self._subsystem_info, component, build_file, all_parts, overrided_components, component_override_map): 916 overrided = True 917 918 if overrided: 919 # Update parts.json and parts_config.json generated by preloader 920 write_json_file(parts_file, {"parts": all_parts}) 921 parts_file = self.platforms_config_file.replace( 922 "platforms.build", "parts_config.json") 923 self._output_parts_config_json(all_parts, parts_file) 924 all_component_override_map.update(component_override_map) 925 write_json_file( 926 f"{self.config_output_dir}/component_override_map.json", all_component_override_map) 927 return overrided_components 928 929 def _override_one_component(self, subsystem_info: dict, component: dict, build_file: str, all_parts: dict, overrided_components: dict, component_override_map: dict): 930 '''Description: Perform a replacement of a single component and return the component list update result. 931 @parameter:subsystem_info, component, build_file, all_parts, overrided_components 932 @return :True or False(Whether replacement has been performed) 933 ''' 934 splits = component["override"].split(":") 935 if len(splits) != 2: 936 LogUtil.hb_warning( 937 "{} override value is invalid format. Skip override process".format(build_file)) 938 return False 939 overrided_subsystem = splits[0] 940 overrided_component = splits[1] 941 if overrided_subsystem not in subsystem_info: 942 LogUtil.hb_warning( 943 "{} override invalid subsystem. Skip override process".format(build_file)) 944 return False 945 946 founded_bundle = "" 947 948 for bundle in subsystem_info[overrided_subsystem]["build_files"]: 949 if not bundle.endswith('bundle.json'): 950 continue 951 952 bundle_obj = read_json_file(bundle) 953 954 if bundle_obj.get("component", {}).get("name") == overrided_component: 955 founded_bundle = bundle 956 break 957 958 if founded_bundle: 959 origin_component = read_json_file(build_file).get('component') 960 LogUtil.hb_warning( 961 f"You are trying to override \"{component['override']}\" with \"{origin_component.get('subsystem')}:{origin_component.get('name')}\". \nPlease ensure that the modules in \"{component['override']}\" only rely on the interfaces of other components through \"external_deps\"") 962 963 # replace bundle.json in subsystem_info's build_files 964 subsystem_info[overrided_subsystem]["build_files"].remove( 965 founded_bundle) 966 967 # Update parts.json generated by preloader, which means that new added components will not be installed 968 # Ensure that the overrided components will be installed 969 full_partname = f"{overrided_subsystem}:{overrided_component}" 970 if full_partname in all_parts: 971 all_parts.remove(full_partname) 972 973 overrided_components[f"{component['subsystem']}:{component['name']}"] = { 974 'subsystem': overrided_subsystem, 975 'partName': overrided_component 976 } 977 component_override_map[overrided_component] = component["name"] 978 return True 979 LogUtil.hb_warning( 980 "{}:{} is not configured in product, \new add component will be installed!".format( 981 overrided_subsystem, overrided_component)) 982 return False 983 984 def _output_parts_config_json(self, all_parts: dict, output_file: dict): 985 '''Description: Update the parts list file generated by preloader 986 @parameter: all_parts, output_file 987 @return :none 988 ''' 989 parts_config = {} 990 for part in all_parts: 991 part = part.replace(":", "_") 992 part = part.replace("-", "_") 993 part = part.replace(".", "_") 994 part = part.replace("/", "_") 995 parts_config[part] = True 996 write_json_file(output_file, parts_config) 997