• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4#
5# Copyright (c) 2023 Huawei Device Co., Ltd.
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18import os
19
20from services.interface.load_interface import LoadInterface
21from containers.status import throw_exception
22from exceptions.ohos_exception import OHOSException
23from util.loader import platforms_loader  # noqa: E402
24from util.loader import generate_targets_gn  # noqa: E402
25from util.loader import load_ohos_build  # noqa: E402
26from util.loader import subsystem_scan  # noqa: E402
27from util.loader import subsystem_info  # noqa: E402
28from scripts.util.file_utils import read_json_file, write_json_file, write_file  # noqa: E402, E501
29from util.log_util import LogUtil
30
31
32class OHOSLoader(LoadInterface):
33
34    def __init__(self):
35        super().__init__()
36        self.source_root_dir = ""
37        self.gn_root_out_dir = ""
38        self.os_level = ""
39        self.target_cpu = ""
40        self.target_os = ""
41        self.config_output_relpath = ""
42        self.config_output_dir = ""
43        self.target_arch = ""
44        self.subsystem_config_file = ""
45        self.subsystem_config_overlay_file = ""
46        self.platforms_config_file = ""
47        self.exclusion_modules_config_file = ""
48        self.example_subsystem_file = ""
49        self.build_example = ""
50        self.scalable_build = ""
51        self.build_platform_name = ""
52        self.build_xts = ""
53        self.ignore_api_check = ""
54        self.load_test_config = ""
55        self.subsystem_configs = ""
56        self._subsystem_info = ""
57        self.skip_partlist_check = ""
58
59    def __post_init__(self):
60        self.source_root_dir = self.config.root_path + '/'
61        self.gn_root_out_dir = self.config.out_path if not self.config.out_path.startswith(
62            '/') else os.path.relpath(self.config.out_path, self.config.root_path)
63        self.os_level = self.config.os_level if self.config.os_level else "standard"
64        self.target_cpu = self.config.target_cpu if self.config.target_cpu else "arm"
65        self.target_os = self.config.target_os if self.config.target_os else "ohos"
66        self.config_output_relpath = os.path.join(
67            self.gn_root_out_dir, 'build_configs')
68        self.config_output_dir = os.path.join(
69            self.source_root_dir, self.config_output_relpath)
70        self.target_arch = '{}_{}'.format(self.target_os, self.target_cpu)
71        self.subsystem_config_file = os.path.join(
72            self.config.root_path, 'out/preloader', self.config.product, 'subsystem_config.json')
73        self.platforms_config_file = os.path.join(
74            self.config.root_path, 'out/preloader', self.config.product, 'platforms.build')
75        self.exclusion_modules_config_file = os.path.join(
76            self.config.root_path, 'out/preloader', self.config.product, 'exclusion_modules.json')
77        self.example_subsystem_file = os.path.join(
78            self.config.root_path, 'build', 'subsystem_config_example.json')
79
80        compile_standard_allow_file = os.path.join(
81            self.config.root_path, 'out/preloader', self.config.product, 'compile_standard_whitelist.json')
82        compile_standard_allow_info = read_json_file(compile_standard_allow_file)
83        bundle_subsystem_allow_list = compile_standard_allow_info.get("bundle_subsystem_error", [])
84
85        # check config args
86        self._check_args()
87
88        self.build_example = self.args_dict.get('build_example')
89        if not self.build_example:
90            self.example_subsystem_file = ""
91        self.scalable_build = self.args_dict.get('scalable_build')
92        self.build_platform_name = self.args_dict.get('build_platform_name')
93        self.build_xts = self.args_dict.get('build_xts')
94        self.ignore_api_check = self.args_dict.get('ignore_api_check')
95        self.load_test_config = self.args_dict.get('load_test_config')
96        self.skip_partlist_check = self.args_dict.get('skip_partlist_check')
97        self.subsystem_configs = subsystem_scan.scan(self.subsystem_config_file,
98                                                     self.example_subsystem_file,
99                                                     self.source_root_dir)
100
101        self._subsystem_info = subsystem_info.get_subsystem_info(
102            self.subsystem_config_file,
103            self.example_subsystem_file,
104            self.source_root_dir,
105            self.config_output_relpath,
106            self.os_level)
107        overrided_components = self._override_components()
108
109        self._platforms_info = platforms_loader.get_platforms_info(
110            self.platforms_config_file,
111            self.source_root_dir,
112            self.gn_root_out_dir,
113            self.target_arch,
114            self.config_output_relpath,
115            self.scalable_build)
116        self.variant_toolchains = self._platforms_info.get(
117            'variant_toolchain_info').get('platform_toolchain')
118        self._all_platforms = self.variant_toolchains.keys()
119        self.build_platforms = self._get_build_platforms()
120        self.parts_config_info = load_ohos_build.get_parts_info(
121            self.source_root_dir,
122            self.config_output_relpath,
123            self._subsystem_info,
124            self.variant_toolchains,
125            self.target_arch,
126            self.ignore_api_check,
127            self.exclusion_modules_config_file,
128            self.load_test_config,
129            overrided_components,
130            bundle_subsystem_allow_list,
131            self.skip_partlist_check,
132            self.build_xts)
133        self.parts_targets = self.parts_config_info.get('parts_targets')
134        self.phony_targets = self.parts_config_info.get('phony_target')
135        self.parts_info = self.parts_config_info.get('parts_info')
136        self.target_platform_parts = self._get_platforms_all_parts()
137        self.target_platform_stubs = self._get_platforms_all_stubs()
138        self.required_parts_targets_list = self._get_required_build_parts_list()
139        self.required_phony_targets = self._get_required_phony_targets()
140        self.required_parts_targets = self._get_required_build_targets()
141
142# check method
143
144    '''Description: Check the parameters passed in config. If the parameters are not
145                    specified or the file content pointed to by the parameters does not
146                    exist, an exception will be thrown directly.
147    @parameter:none
148    @return :none
149    '''
150    @throw_exception
151    def _check_args(self):
152        LogUtil.hb_info("Checking all build args...")
153        # check subsystem_config_file
154        if not read_json_file(self.subsystem_config_file):
155            self.subsystem_config_file = os.path.join(
156                self.source_root_dir, 'build/subsystem_config.json')
157        if not read_json_file(self.subsystem_config_file):
158            raise OHOSException("Cannot get the content from platform config file, \
159                            please check whether the corresponding file('out/preloader/{}/subsystem_config.json' or \
160                            'build/subsystem_config.json') is written correctly.".format(self.config.product), "2001")
161
162        # check gn_root_out_dir
163        if not self.gn_root_out_dir:
164            raise OHOSException("Args gn_root_out_dir is required.", "2002")
165        if not os.path.realpath(self.gn_root_out_dir).startswith(self.source_root_dir):
166            raise OHOSException("Args gn_root_out_dir is incorrect.", "2003")
167
168        # check platform config file
169        if not read_json_file(self.platforms_config_file):
170            raise OHOSException("Cannot get the content from platform config file, \
171                            please check whether the corresponding file('out/preloader/${product_name}/platforms.build') \
172                            is written correctly.".format(self.config.product), "2004")
173
174        # check example subsystem file
175        if not read_json_file(self.example_subsystem_file):
176            raise OHOSException("Cannot get the content from example subsystem file, please check whether \
177                                the corresponding file ('build/subsystem_config_example.json') exists.", "2005")
178
179    @throw_exception
180    def _check_product_part_feature(self):
181        LogUtil.hb_info("Checking all product features...")
182        product_preloader_dir = os.path.dirname(self.platforms_config_file)
183        _preloader_feature_file = os.path.join(product_preloader_dir,
184                                               'features.json')
185        _preloader_feature_info = read_json_file(_preloader_feature_file)
186        part_to_feature = _preloader_feature_info.get('part_to_feature')
187        for key, vals in part_to_feature.items():
188            part = self.parts_info.get(key)
189            if part is None:
190                continue
191            _p_info = part[0]
192            def_feature_list = _p_info.get('feature_list')
193            if not def_feature_list:
194                continue
195            for _f_name in vals:
196                if _f_name not in def_feature_list:
197                    raise OHOSException(
198                        "The product use a feature that is not supported"
199                        " by this part, part_name='{}', feature='{}'".format(
200                            key, _f_name), "2006")
201
202    @throw_exception
203    def _check_parts_config_info(self):
204        LogUtil.hb_info("Checking parts config...")
205        if not ('parts_info' in self.parts_config_info
206                and 'subsystem_parts' in self.parts_config_info
207                and 'parts_variants' in self.parts_config_info
208                and 'parts_kits_info' in self.parts_config_info
209                and 'parts_inner_kits_info' in self.parts_config_info
210                and 'parts_targets' in self.parts_config_info):
211            raise OHOSException(
212                "Loading ohos.build information is incorrect.", "2007")
213
214# generate method
215
216    '''Description: Generate SystemCapability.json & syscap.json & syscap.para, dir:[
217        (//out/preloader/${product_name}/system/etc/SystemCapability.json),
218        (//out/preloader/${product_name}/system/etc/syscap.json),
219        (//out/preloader/${product_name}/system/etc/param/syscap.para)]
220    @parameter:none
221    @return :none
222    '''
223    @throw_exception
224    def _generate_syscap_files(self):
225        pre_syscap_info_path = os.path.dirname(self.platforms_config_file)
226        system_path = os.path.join(self.source_root_dir, os.path.join(
227            os.path.dirname(self.platforms_config_file), "system/"))
228        syscap_product_dict = read_json_file(
229            os.path.join(pre_syscap_info_path, "syscap.json"))
230        syscap_info_list = self.parts_config_info.get('syscap_info')
231        target_syscap_with_part_name_list = []
232        target_syscap_list = []
233        target_syscap_for_init_list = []
234        all_syscap_list = []
235        for syscap in syscap_info_list:
236            if syscap['component'] not in self.required_parts_targets_list:
237                continue
238            if 'syscap' not in syscap or syscap['syscap'] is None \
239                    or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]:
240                continue
241            for syscap_string in syscap['syscap']:
242                all_syscap_list.append(syscap_string.split('=')[0].strip())
243
244        for key, value in syscap_product_dict['part_to_syscap'].items():
245            for syscap in value:
246                if syscap not in all_syscap_list:
247                    raise OHOSException(
248                        "In config.json of part [{}],the syscap[{}] is incorrect, \
249                        please check the syscap name".format(key, syscap), "2008")
250
251        for syscap in syscap_info_list:
252            remove_list = []
253            if syscap['component'] not in self.required_parts_targets_list:
254                continue
255            if 'syscap' not in syscap or syscap['syscap'] is None \
256                    or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]:
257                continue
258            for syscap_string in syscap['syscap']:
259                if syscap_string.startswith("SystemCapability.") is True:
260                    target_syscap_init_str = "const."
261                    syscap_name = syscap_string.split('=')[0].strip()
262                    all_syscap_product = syscap_product_dict['syscap']
263                    if syscap_name in all_syscap_product and not all_syscap_product[syscap_name]:
264                        remove_list.append(syscap_string)
265                        continue
266                    elif syscap_name in all_syscap_product and all_syscap_product[syscap_name]:
267                        target_syscap_init_str += syscap_name + '=true\n'
268                    else:
269                        if syscap_string.endswith('true'):
270                            target_syscap_init_str += syscap_name + '=true\n'
271                        elif syscap_string.endswith('false'):
272                            remove_list.append(syscap_string)
273                            continue
274                        else:
275                            target_syscap_init_str += syscap_string + "=true\n"
276                    if target_syscap_init_str not in target_syscap_for_init_list:
277                        target_syscap_for_init_list.append(
278                            target_syscap_init_str)
279                else:
280                    raise OHOSException("""In bundle.json of part [{}], The syscap string [{}] is incorrect,
281                    need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string), "2009")
282
283            for remove_str in remove_list:
284                syscap['syscap'].remove(remove_str)
285            for i in range(len(syscap['syscap'])):
286                if syscap['syscap'][i].endswith('true') or syscap['syscap'][i].endswith('false'):
287                    syscap['syscap'][i] = syscap['syscap'][i].split('=')[
288                        0].strip()
289
290            syscap['syscap'].sort()
291            target_syscap_with_part_name_list.append(syscap)
292            target_syscap_list.extend(syscap['syscap'])
293
294        # Generate SystemCapability.json & syscap.json & syscap.para
295        target_syscap_list.sort()
296        syscap_info_dict = read_json_file(os.path.join(
297            pre_syscap_info_path, "SystemCapability.json"))
298        syscap_info_dict.update({'syscap': {'os': target_syscap_list}})
299        system_etc_path = os.path.join(system_path, "etc/")
300        if not os.path.exists(system_path):
301            os.mkdir(system_path)
302        if not os.path.exists(system_etc_path):
303            os.mkdir(system_etc_path)
304        syscap_info_json = os.path.join(
305            system_etc_path, "SystemCapability.json")
306        write_json_file(syscap_info_json, syscap_info_dict)
307        LogUtil.hb_info(
308            "generate syscap info file to '{}'".format(syscap_info_json))
309        target_syscap_with_part_name_list.sort(
310            key=lambda syscap: syscap['component'])
311        syscap_info_with_part_name_file = os.path.join(
312            system_etc_path, "syscap.json")
313        write_json_file(syscap_info_with_part_name_file, {
314            'components': target_syscap_with_part_name_list})
315        LogUtil.hb_info("generate syscap info with part name list to '{}'".format(
316            syscap_info_with_part_name_file))
317        if not os.path.exists(os.path.join(system_etc_path, "param/")):
318            os.mkdir(os.path.join(system_etc_path, "param/"))
319        target_syscap_for_init_file = os.path.join(
320            system_etc_path, "param/syscap.para")
321        with open(target_syscap_for_init_file, "w") as file:
322            file.writelines(target_syscap_for_init_list)
323        LogUtil.hb_info("generate target syscap for init list to '{}'".format(
324            target_syscap_for_init_file))
325
326    '''Description: output infos for testfwk into a json file. \
327        (/out/${product_name}/build_configs/infos_for_testfwk.json)
328    @parameter:none
329    @return :none
330    '''
331
332    def _generate_infos_for_testfwk(self):
333        infos_for_testfwk_file = os.path.join(self.config_output_dir,
334                                              "infos_for_testfwk.json")
335        parts_info = self.parts_config_info.get('parts_info')
336        parts_info_dict = {}
337        for _part_name, _parts in parts_info.items():
338            for _info in _parts:
339                parts_info_dict[_info.get('part_name')] = _info
340        _output_infos = {}
341        for _platform, _parts in self.target_platform_parts.items():
342            result = self._output_infos_by_platform(_parts, parts_info_dict)
343            _output_infos[_platform] = result
344        write_json_file(infos_for_testfwk_file,
345                        _output_infos, check_changes=True)
346        LogUtil.hb_info("generate infos for testfwk to '{}'".format(
347            infos_for_testfwk_file))
348
349    '''Description: output all target platform parts into a json file \
350        (/out/${product_name}/build_configs/target_platforms_parts.json)
351    @parameter:none
352    @return :none
353    '''
354
355    def _generate_target_platform_parts(self):
356        target_platform_parts_file = os.path.join(self.config_output_dir,
357                                                  "target_platforms_parts.json")
358        write_json_file(target_platform_parts_file,
359                        self.target_platform_parts,
360                        check_changes=True)
361        LogUtil.hb_info("generate target platform parts to '{}'".format(
362            target_platform_parts_file))
363
364    '''Description: Generate parts differences in different platforms, using phone as base. \
365        (/out/${product_name}/build_configs/parts_different_info.json)
366    @parameter: none
367    @return :none
368    '''
369
370    def _generate_part_different_info(self):
371        parts_different_info = self._get_parts_by_platform()
372        parts_different_info_file = os.path.join(self.config_output_dir,
373                                                 "parts_different_info.json")
374        write_json_file(parts_different_info_file,
375                        parts_different_info,
376                        check_changes=True)
377        LogUtil.hb_info("generate part different info to '{}'".format(
378            parts_different_info_file))
379
380    '''Description: output platforms list into a gni file. \
381        (/out/${product_name}/build_configs/platforms_list.gni)
382    @parameter: none
383    @return: none
384    '''
385
386    def _generate_platforms_list(self):
387        platforms_list_gni_file = os.path.join(self.config_output_dir,
388                                               "platforms_list.gni")
389        _platforms = set(self.build_platforms)
390        _gni_file_content = ['target_platform_list = [', '  "{}"'.format('",\n  "'.join(_platforms)), ']',
391                             'kits_platform_list = [', '  "{}",'.format('",\n  "'.join(_platforms))]
392        if 'phone' not in self.build_platforms:
393            _gni_file_content.append('  "phone"')
394        _gni_file_content.append(']')
395        write_file(platforms_list_gni_file, '\n'.join(_gni_file_content))
396        LogUtil.hb_info("generate platforms list to '{}'".format(
397            platforms_list_gni_file))
398
399    '''Description: output auto install part into a json file. \
400        (/out/${product_name}/build_configs/auto_install_parts.json)
401    @parameter: none
402    @return: none
403    '''
404
405    def _generate_auto_install_part(self):
406        parts_path_info = self.parts_config_info.get("parts_path_info")
407        auto_install_part_list = []
408        for part, path in parts_path_info.items():
409            if str(path).startswith("drivers/interface") or \
410                    str(path).startswith("third_party"):
411                auto_install_part_list.append(part)
412        auto_install_list_file = os.path.join(
413            self.config_output_dir, "auto_install_parts.json")
414        write_json_file(auto_install_list_file, auto_install_part_list)
415        LogUtil.hb_info("generate auto install part to '{}'".format(
416            auto_install_list_file))
417
418    '''Description: output src flag into a json file. \
419        (/out/${product_name}/build_configs/parts_src_flag.json)
420    @parameter: none
421    @return :none
422    '''
423
424    def _generate_src_flag(self):
425        parts_src_flag_file = os.path.join(self.config_output_dir,
426                                           "parts_src_flag.json")
427        write_json_file(parts_src_flag_file,
428                        self._get_parts_src_list(),
429                        check_changes=True)
430        LogUtil.hb_info(
431            "generated parts src flag to '{}/subsystem_info/parts_src_flag.json'".format(self.config_output_dir))
432
433    '''Description: output build target list into a json file.\
434        (/out/${product_name}/build_configs/required_parts_targets_list.json)
435    @parameter: none
436    @return :none
437    '''
438
439    def _generate_required_parts_targets_list(self):
440        build_targets_list_file = os.path.join(self.config_output_dir,
441                                               "required_parts_targets_list.json")
442        write_json_file(build_targets_list_file,
443                        list(self.required_parts_targets.values()))
444        LogUtil.hb_info("generate build targets list file to '{}'".format(
445            build_targets_list_file))
446
447    '''Description: output build target info into a json file. \
448        (/out/${product_name}/build_configs/required_parts_targets.json)
449    @parameter: none
450    @return: none
451    '''
452
453    def _generate_required_parts_targets(self):
454        build_targets_info_file = os.path.join(self.config_output_dir,
455                                               "required_parts_targets.json")
456        write_json_file(build_targets_info_file, self.required_parts_targets)
457        LogUtil.hb_info("generate required parts targets to '{}'".format(
458            build_targets_info_file))
459
460    '''Description: output platforms part by src into a json file. \
461        (/out/${product_name}/build_configs/platforms_parts_by_src.json)
462    @parameter: none
463    @return :none
464    '''
465
466    def _generate_platforms_part_by_src(self):
467        platforms_parts_by_src = self._get_platforms_parts()
468        platforms_parts_by_src_file = os.path.join(self.source_root_dir,
469                                                   self.config_output_relpath,
470                                                   "platforms_parts_by_src.json")
471        write_json_file(platforms_parts_by_src_file,
472                        platforms_parts_by_src,
473                        check_changes=True)
474        LogUtil.hb_info("generated platforms parts by src to '{}'".format(
475            platforms_parts_by_src_file))
476
477    '''Description: output system configs info into 4 files:[
478        (/out/${product_name}/build_configs/subsystem_info/parts_list.gni),
479        (/out/${product_name}/build_configs/subsystem_info/inner_kits_list.gni),
480        (/out/${product_name}/build_configs/subsystem_info/system_kits_list.gni),
481        (/out/${product_name}/build_configs/subsystem_info/parts_test_list.gni),
482        (/out/${product_name}/build_configs/subsystem_info/BUILD.gn)]
483    @parameter: none
484    @return :none
485    '''
486
487    def _generate_target_gn(self):
488        generate_targets_gn.gen_targets_gn(self.required_parts_targets,
489                                           self.config_output_dir)
490
491    '''Description: output phony targets build file. \
492        (/out/${product_name}/build_configs/phony_target/BUILD.gn)
493    @parameter: none
494    @return :none
495    '''
496
497    def _generate_phony_targets_build_file(self):
498        generate_targets_gn.gen_phony_targets(self.required_phony_targets,
499                                              self.config_output_dir)
500
501    '''Description: output system configs info into 2 files:[
502        (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/BUILG.gn),
503        (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/zframework_stub_exists.gni)]
504    @parameter: none
505    @return :none
506    '''
507
508    def _generate_stub_targets(self):
509        generate_targets_gn.gen_stub_targets(
510            self.parts_config_info.get('parts_kits_info'),
511            self.target_platform_stubs,
512            self.config_output_dir)
513
514    '''Description: output system capabilities into a json file. \
515        (/out/${product_name}/build_configs/${platform}_system_capabilities.json)
516    @parameter: none
517    @return :none
518    '''
519
520    def _generate_system_capabilities(self):
521        for platform in self.build_platforms:
522            platform_parts = self.target_platform_parts.get(platform)
523            platform_capabilities = []
524            for _, origin in platform_parts.items():
525                # parts_info.get() might be None if the part is a binary package
526                all_parts_variants = self.parts_info.get(origin)
527                if all_parts_variants is None:
528                    continue
529                part = all_parts_variants[0]
530                if part.get('system_capabilities'):
531                    entry = part.get('system_capabilities')
532                    if len(entry) > 0:
533                        platform_capabilities.extend(entry)
534            platform_part_json_file = os.path.join(
535                self.config_output_dir, "{0}_system_capabilities.json".format(platform))
536            write_json_file(platform_part_json_file,
537                            sorted(platform_capabilities),
538                            check_changes=True)
539            LogUtil.hb_info(
540                "generated system capabilities to '{}/{}_system_capabilities.json'".format(
541                    self.config_output_dir, platform))
542
543    '''Description: output system configs info into three json files:[
544        (/out/${product_name}/build_configs/subsystem_info/subsystem_build_config.json),
545        (/out/${product_name}/build_configs/subsystem_info/src_subsystem_info.json),
546        (/out/${product_name}/build_configs/subsystem_info/no_src_subsystem_info.json)]
547    @parameter: none
548    @return :none
549    '''
550
551    def _generate_subsystem_configs(self):
552
553        # The function has been implemented in module util/loader/subsystem_info.py
554        LogUtil.hb_info(
555            "generated subsystem build config to '{}/subsystem_info/subsystem_build_config.json'".format(
556                self.config_output_dir))
557        LogUtil.hb_info(
558            "generated src subsystem info to '{}/subsystem_info/src_subsystem_info.json'".format(
559                self.config_output_dir))
560        LogUtil.hb_info(
561            "generated no src subsystem info to '{}/subsystem_info/no_src_subsystem_info.json'".format(
562                self.config_output_dir))
563
564# get method
565    @throw_exception
566    def _get_build_platforms(self) -> list:
567        build_platforms = []
568        if self.build_platform_name == 'all':
569            build_platforms = self._all_platforms
570        elif self.build_platform_name in self._all_platforms:
571            build_platforms = [self.build_platform_name]
572        else:
573            raise OHOSException(
574                "The target_platform is incorrect, only allows [{}].".format(
575                    ', '.join(self._all_platforms)), "2010")
576        return build_platforms
577
578    def _get_parts_by_platform(self) -> dict:
579        parts_info = {}
580        if 'phone' in self.target_platform_parts:
581            phone_parts_list = self.target_platform_parts.get('phone').keys()
582        else:
583            phone_parts_list = []
584        for _platform, _parts_info in self.target_platform_parts.items():
585            base_parts_list = []
586            curr_parts_list = []
587            for _real_name, _original_name in _parts_info.items():
588                if _real_name in phone_parts_list:
589                    base_parts_list.append(_real_name)
590                elif _original_name in phone_parts_list:
591                    base_parts_list.append(_real_name)
592                else:
593                    curr_parts_list.append(_real_name)
594            result_data = {
595                "base_parts_list": base_parts_list,
596                "curr_parts_list": curr_parts_list
597            }
598            parts_info[_platform] = result_data
599        return parts_info
600
601    def _get_platforms_all_parts(self) -> dict:
602        _dist_parts_variants = self._load_component_dist()
603        target_platform_parts = {}
604        all_parts = self._platforms_info.get('all_parts')
605        parts_variants = self.parts_config_info.get('parts_variants')
606        for _platform, _parts in all_parts.items():
607            if _platform not in self.build_platforms:
608                continue
609            part_name_info = {}
610            for part_def in _parts:
611                real_name, original_name = self._get_real_part_name(
612                    part_def, _platform, parts_variants)
613                if real_name is None:
614                    # find this from component_dist
615                    real_name, original_name = self._get_real_part_name(
616                        part_def, _platform, _dist_parts_variants)
617                if real_name is None:
618                    continue
619                part_name_info[real_name] = original_name
620            target_platform_parts[_platform] = part_name_info
621        return target_platform_parts
622
623    def _get_platforms_all_stubs(self) -> dict:
624        _dist_parts_variants = self._load_component_dist()
625        platform_stubs = {}
626        all_stubs = self._platforms_info.get('all_stubs')
627        parts_variants = self.parts_config_info.get('parts_variants')
628        for _platform, _part_names in all_stubs.items():
629            if _platform not in self.build_platforms:
630                continue
631            stub_parts_from_src = []
632            stub_parts_from_dist = []
633            for part_name in _part_names:
634                real_name, original_name = self._get_real_part_name(
635                    part_name, _platform, parts_variants)
636                # real_name=None means part_name doesn't exist in source tree,
637                # use binary in component_dist then.
638                if real_name is None:
639                    # find this from component_dist
640                    real_name, original_name = self._get_real_part_name(
641                        part_name, _platform, _dist_parts_variants)
642                    if real_name is None:
643                        continue
644                    else:
645                        stub_sources = os.path.join(
646                            self.source_root_dir,
647                            "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt"  # noqa: E501
648                            .format(self.target_os, self.target_cpu, real_name))
649                        stub_parts_from_dist.append(
650                            '"{}"'.format(stub_sources))
651                else:
652                    stub_parts_from_src.append(real_name)
653            platform_stubs[_platform] = {
654                "src": stub_parts_from_src,
655                "dist": stub_parts_from_dist,
656            }
657        return platform_stubs
658
659    def _get_platforms_parts(self) -> dict:
660        platforms_parts = {}
661        src_parts_targets = self.parts_targets
662        src_all_parts = src_parts_targets.keys()
663        for _platform, _all_parts in self.target_platform_parts.items():
664            src_parts_list = []
665            no_src_parts_list = []
666            for _part in _all_parts.keys():
667                if _part in src_all_parts:
668                    src_parts_list.append(_part)
669                else:
670                    no_src_parts_list.append(_part)
671            _data = {
672                'src_parts': src_parts_list,
673                'no_src_parts': no_src_parts_list
674            }
675            platforms_parts[_platform] = _data
676        return platforms_parts
677
678    def _get_parts_src_list(self) -> list:
679        parts_name_map = {}
680        for _list in self.parts_info.values():
681            for _info in _list:
682                parts_name_map[_info.get('part_name')] = _info.get(
683                    'origin_part_name')
684        _src_set = set()
685        for _name in self.required_parts_targets.keys():
686            _origin_name = parts_name_map.get(_name)
687            if _origin_name is None:
688                continue
689            _src_set.add(_origin_name)
690        return list(_src_set)
691
692    def _get_required_build_targets(self) -> dict:
693        required_build_targets = {}
694        for _p_name, _info in self.parts_targets.items():
695            if _p_name not in self.required_parts_targets_list:
696                continue
697            required_build_targets[_p_name] = _info
698        return required_build_targets
699
700    def _get_required_phony_targets(self) -> dict:
701        required_build_targets = {}
702        for _p_name, _info in self.phony_targets.items():
703            if _p_name not in self.required_parts_targets_list:
704                continue
705            required_build_targets[_p_name] = _info
706        return required_build_targets
707
708    def _get_required_build_parts_list(self) -> list:
709        parts_set = set()
710        for _parts_list in self.target_platform_parts.values():
711            parts_set.update(_parts_list)
712        return list(parts_set)
713
714# util method
715
716    def _load_component_dist(self) -> dict:
717        _parts_variants_info = {}
718        _dir = "component_dist/{}-{}/packages_to_install".format(
719            self.target_os, self.target_cpu)
720        _file_name = "dist_parts_info.json"
721        _dist_parts_info_file = os.path.join(
722            self.source_root_dir, _dir, _file_name)
723        if not os.path.exists(_dist_parts_info_file):
724            # If the file does not exist, do nothing and return
725            return _parts_variants_info
726        _parts_info = read_json_file(_dist_parts_info_file)
727        if _parts_info is None:
728            raise Exception("read file '{}' failed.".format(
729                _dist_parts_info_file))
730        for _part_info in _parts_info:
731            origin_part_name = _part_info.get('origin_part_name')
732            if origin_part_name in _parts_variants_info:
733                variants = _parts_variants_info.get(origin_part_name)
734            else:
735                variants = []
736            _variant_name = _part_info.get('variant_name')
737            variants.append(_variant_name)
738            _parts_variants_info[origin_part_name] = variants
739        return _parts_variants_info
740
741    def _get_real_part_name(self, original_part_name, current_platform, parts_variants):
742        part_info = parts_variants.get(original_part_name)
743        if part_info is None:
744            return None, None
745        if current_platform in part_info and current_platform != 'phone':
746            real_name = '{}_{}'.format(original_part_name, current_platform)
747        else:
748            real_name = original_part_name
749        return real_name, original_part_name
750
751    '''Description: called by _out_infos_for_testfwk, output information by platform
752    @parameter:none
753    @return :none
754    '''
755
756    def _output_infos_by_platform(self, part_name_infos, parts_info_dict):
757        required_parts = {}
758        subsystem_infos = {}
759        for part_name, origin_part_name in part_name_infos.items():
760            part_info = parts_info_dict.get(part_name)
761            if part_info is None:
762                continue
763            if origin_part_name != part_info.get('origin_part_name'):
764                raise Exception("part configuration is incorrect.")
765            required_parts[origin_part_name] = part_info
766            _subsystem_name = part_info.get('subsystem_name')
767            if _subsystem_name in subsystem_infos:
768                p_list = subsystem_infos.get(_subsystem_name)
769            else:
770                p_list = []
771            p_list.append(origin_part_name)
772            subsystem_infos[_subsystem_name] = p_list
773        result = {}
774        result['subsystem_infos'] = subsystem_infos
775        result['part_infos'] = required_parts
776        return result
777
778    def _execute_loader_args_display(self):
779        LogUtil.hb_info('Loading configuration file...')
780        args = []
781        args.append('platforms_config_file="{}"'.format(
782            self.platforms_config_file))
783        args.append('subsystem_config_file="{}"'.format(
784            self.subsystem_config_file))
785        args.append('example_subsystem_file="{}"'.format(
786            self.example_subsystem_file))
787        args.append('exclusion_modules_config_file="{}"'.format(
788            self.exclusion_modules_config_file))
789        args.append('source_root_dir="{}"'.format(self.source_root_dir))
790        args.append('gn_root_out_dir="{}"'.format(self.gn_root_out_dir))
791        args.append('build_platform_name={}'.format(self.build_platform_name))
792        args.append('build_xts={}'.format(self.build_xts))
793        args.append('load_test_config={}'.format(self.load_test_config))
794        args.append('target_os={}'.format(self.target_os))
795        args.append('target_cpu={}'.format(self.target_cpu))
796        args.append('os_level={}'.format(self.os_level))
797        args.append('ignore_api_check={}'.format(self.ignore_api_check))
798        args.append('scalable_build={}'.format(self.scalable_build))
799        args.append('skip_partlist_check={}'.format(self.skip_partlist_check))
800        LogUtil.write_log(self.config.log_path,
801                          'loader args:{}'.format(args), 'info')
802
803    def _override_components(self):
804        '''Description: Check whether there are components that need to be replaced, and if so,
805            replace the component configuration file bundle.json in subsystem_info and update
806            the component list generated by the preloader.
807        @parameter:none
808        @return :overrided_components
809        '''
810        parts_file = self.platforms_config_file.replace(
811            "platforms.build", "parts.json")
812        all_parts = read_json_file(parts_file)
813        if "parts" not in all_parts:
814            LogUtil.hb_warning("{} does not contain parts!".format(parts_file))
815            return {}
816        overrided = False
817        overrided_components = {}
818        all_parts = all_parts["parts"]
819        component_override_map = {}
820        for subsystem_name, build_config_info in self._subsystem_info.items():
821            if "build_files" not in build_config_info:
822                continue
823
824            # scan all bundle.json or ohos.build files with named groups
825            for build_file in build_config_info["build_files"]:
826
827                # ohos.build does not support overrided components
828                if not build_file.endswith('bundle.json'):
829                    continue
830
831                # Only device or vendor components can do named groups extensions
832                if (not build_file.startswith(self.source_root_dir + 'device/')) \
833                        and (not build_file.startswith(self.source_root_dir + 'vendor/')):
834                    continue
835
836                # "subsystem", "name" and "override" is required
837                component = read_json_file(build_file).get("component")
838
839                if (not component) or (not all(key in component for key in ("subsystem", "name", "override"))):
840                    continue
841
842                full_part_name = f"{component.get('subsystem')}:{component.get('name')}"
843                if full_part_name not in all_parts:
844                    LogUtil.hb_warning("{} was not configured for this product: {}".format(
845                        build_file, full_part_name))
846                    continue
847
848                if self._override_one_component(self._subsystem_info, component, build_file, all_parts, overrided_components, component_override_map):
849                    overrided = True
850
851        if overrided:
852            # Update parts.json and parts_config.json generated by preloader
853            write_json_file(parts_file, {"parts": all_parts})
854            parts_file = self.platforms_config_file.replace(
855                "platforms.build", "parts_config.json")
856            self._output_parts_config_json(all_parts, parts_file)
857
858        write_json_file(
859            f"{self.config_output_dir}/component_override_map.json", component_override_map)
860        return overrided_components
861
862    def _override_one_component(self, subsystem_info, component, build_file, all_parts, overrided_components, component_override_map):
863        '''Description: Perform a replacement of a single component and return the component list update result.
864        @parameter:subsystem_info, component, build_file, all_parts, overrided_components
865        @return :True or False(Whether replacement has been performed)
866        '''
867        splits = component["override"].split(":")
868        if len(splits) != 2:
869            LogUtil.hb_warning(
870                "{} override value is invalid format. Skip override process".format(build_file))
871            return False
872        overrided_subsystem = splits[0]
873        overrided_component = splits[1]
874        if overrided_subsystem not in subsystem_info:
875            LogUtil.hb_warning(
876                "{} override invalid subsystem. Skip override process".format(build_file))
877            return False
878
879        founded_bundle = ""
880
881        for bundle in subsystem_info[overrided_subsystem]["build_files"]:
882            if not bundle.endswith('bundle.json'):
883                continue
884
885            bundle_obj = read_json_file(bundle)
886
887            if bundle_obj.get("component", {}).get("name") == overrided_component:
888                founded_bundle = bundle
889                break
890
891        if founded_bundle:
892            origin_component = read_json_file(build_file).get('component')
893            LogUtil.hb_warning(
894                f"You are trying to override \"{component['override']}\" with \"{origin_component.get('subsystem')}:{origin_component.get('name')}\". \nPlease ensure that the modules in \"{component['override']}\" only rely on the interfaces of other components through \"external_deps\"")
895
896            # replace bundle.json in subsystem_info's build_files
897            subsystem_info[overrided_subsystem]["build_files"].remove(
898                founded_bundle)
899
900            # Update parts.json generated by preloader, which means that new added components will not be installed
901            # Ensure that the overrided components will be installed
902            full_partname = f"{overrided_subsystem}:{overrided_component}"
903            if full_partname in all_parts:
904                all_parts.remove(full_partname)
905
906            overrided_components[f"{component['subsystem']}:{component['name']}"] = {
907                'subsystem': overrided_subsystem,
908                'partName': overrided_component
909            }
910            component_override_map[overrided_component] = component["name"]
911            return True
912        LogUtil.hb_warning(
913            "{}:{} is not configured in product, \new add component will be installed!".format(
914                overrided_subsystem, overrided_component))
915        return False
916
917    def _output_parts_config_json(self, all_parts, output_file):
918        '''Description: Update the parts list file generated by preloader
919        @parameter: all_parts, output_file
920        @return :none
921        '''
922        parts_config = {}
923        for part in all_parts:
924            part = part.replace(":", "_")
925            part = part.replace("-", "_")
926            part = part.replace(".", "_")
927            part = part.replace("/", "_")
928            parts_config[part] = True
929        write_json_file(output_file, parts_config)
930