• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#
4# Copyright (c) 2025 Huawei Device Co., Ltd.
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import logging
19import os
20from pathlib import Path
21import shutil
22import subprocess
23from typing import List, Mapping
24import binascii
25import glob
26import tarfile
27
28
29# 配置日志记录
30logging.basicConfig(
31    level=logging.INFO,
32    format='%(asctime)s - %(levelname)s - %(message)s',
33    handlers=[
34        logging.StreamHandler()
35    ]
36)
37
38
39def run_command(cmd, cwd=None, env=None):
40    logger = logging.getLogger(__name__)
41    try:
42        logger.info(f"Command: {' '.join(cmd)}")
43        result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=True)
44        if result.stdout:
45            logger.info(f"Command output: {result.stdout.strip()}")
46        if result.stderr:
47            logger.warning(f"Command error output: {result.stderr.strip()}")
48    except subprocess.CalledProcessError as e:
49        logger.error(f"Command failed: {' '.join(cmd)}. Error: {e.stderr.strip()}")
50        raise
51
52class BuildConfig:
53    def __init__(self, args):
54        self.REPOROOT_DIR = args.repo_root
55        self.OUT_PATH = args.out_path
56        self.LLDB_PY_VERSION = args.lldb_py_version
57        self.LLDB_PY_DETAILED_VERSION = args.lldb_py_detailed_version
58        self.MINGW_TRIPLE = args.mingw_triple
59
60
61class PythonBuilder:
62    target_platform = ""
63    patches = []
64
65
66    def __init__(self, build_config) -> None:
67        self.build_config = build_config
68        self.repo_root = Path(build_config.REPOROOT_DIR).resolve()
69        self._out_dir = Path(build_config.OUT_PATH).resolve()
70        self._lldb_py_version = build_config.LLDB_PY_VERSION
71        self._version = build_config.LLDB_PY_DETAILED_VERSION
72        version_parts = self._version.split('.')
73        self._major_version = version_parts[0]
74        self._source_dir = self.repo_root / 'third_party' / 'python'
75        self._patch_dir = self._source_dir / '.cid' / 'patches'
76        self._prebuilts_path = os.path.join(self.repo_root, 'prebuilts')
77        self._prebuilts_python_path = os.path.join(self._prebuilts_path, 'python', 'linux-x86', self._lldb_py_version, 'bin',
78                                                  f'python{self._major_version}')
79        self._install_dir = ""
80        self._clean_patches()
81        logging.getLogger(__name__).addHandler(logging.FileHandler(self._out_dir / 'build.log'))
82
83    @property
84    def _logger(self) -> logging.Logger:
85        return logging.getLogger(__name__)
86
87    @property
88    def _cc(self) -> Path:
89        return self._clang_toolchain_dir / 'bin' / 'clang'
90
91    @property
92    def _cflags(self) -> List[str]:
93        return []
94
95    @property
96    def _ldflags(self) -> List[str]:
97        return []
98
99    @property
100    def _cxx(self) -> Path:
101        return self._clang_toolchain_dir / 'bin' / 'clang++'
102
103    @property
104    def _strip(self) -> Path:
105        return self._clang_toolchain_dir / 'bin' / 'llvm-strip'
106
107    @property
108    def _cxxflags(self) -> List[str]:
109        return self._cflags.copy()
110
111    @property
112    def _rcflags(self) -> List[str]:
113        return []
114
115    @property
116    def _env(self) -> Mapping[str, str]:
117        env = os.environ.copy()
118        clang_bin_dir = self._clang_toolchain_dir / 'bin'
119
120        env.update({
121            'CC': str(self._cc),
122            'CXX': str(self._cxx),
123            'WINDRES': str(clang_bin_dir / 'llvm-windres'),
124            'AR': str(clang_bin_dir / 'llvm-ar'),
125            'READELF': str(clang_bin_dir / 'llvm-readelf'),
126            'LD': str(clang_bin_dir / 'ld.lld'),
127            'DLLTOOL': str(clang_bin_dir / 'llvm-dlltoo'),
128            'RANLIB': str(clang_bin_dir / 'llvm-ranlib'),
129            'STRIP': str(self._strip),
130            'CFLAGS': ' '.join(self._cflags),
131            'CXXFLAGS': ' '.join(self._cxxflags),
132            'LDFLAGS': ' '.join(self._ldflags),
133            'RCFLAGS': ' '.join(self._rcflags),
134            'CPPFLAGS': ' '.join(self._cflags),
135            'LIBS': '-lffi'
136        })
137        return env
138
139    def _configure(self) -> None:
140        self._logger.info("Starting configuration...")
141        return
142
143    def _clean_patches(self) -> None:
144        self._logger.info("Cleaning patches...")
145        run_command(['git', 'reset', '--hard', 'HEAD'], cwd=self._source_dir)
146        run_command(['git', 'clean', '-df', '--exclude=.cid'], cwd=self._source_dir)
147
148    def _pre_build(self) -> None:
149        self._deps_build()
150        self._apply_patches()
151
152    def _apply_patches(self) -> None:
153      if hasattr(self, '_patch_ignore_file') and self._patch_ignore_file.is_file():
154          self._logger.warning('Patches for Python have being applied, skip patching')
155          return
156
157      if not self._patch_dir.is_dir():
158          self._logger.warning('Patches are not found, skip patching')
159          return
160
161      for patch in self._patch_dir.iterdir():
162          if patch.is_file() and patch.name in self.patches:
163              cmd = ['git', 'apply', str(patch)]
164              self._logger.info(f"Applying patch: {patch.name}")
165              run_command(cmd, cwd=self._source_dir)
166
167
168    def _deps_build(self) -> None:
169        self._logger.info("Starting dependency build process...")
170        return
171
172    def build(self) -> None:
173        self._logger.info("Starting build process...")
174        self._pre_build()
175        if hasattr(self, '_build_dir') and self._build_dir.exists():
176            self._logger.info(f"Removing existing build directory: {self._build_dir}")
177            shutil.rmtree(self._build_dir)
178        if isinstance(self._install_dir, Path) and self._install_dir.exists():
179            self._logger.info(f"Removing existing install directory: {self._install_dir}")
180            shutil.rmtree(self._install_dir)
181        if hasattr(self, '_build_dir'):
182            self._build_dir.mkdir(parents=True)
183        if isinstance(self._install_dir, Path):
184            self._install_dir.mkdir(parents=True)
185        self._configure()
186        self._install()
187
188    def _install(self) -> None:
189        self._logger.info("Starting installation...")
190        num_jobs = os.cpu_count() or 8
191        cmd = ['make', f'-j{num_jobs}', 'install']
192        run_command(cmd, cwd=self._build_dir)
193
194    def _strip_in_place(self, file: Path) -> None:
195        self._logger.info(f"Stripping file: {file}")
196        cmd = [
197            str(self._strip),
198            str(file),
199        ]
200        run_command(cmd)
201
202    def _clean_bin_dir(self) -> None:
203        self._logger.info("Cleaning bin directory...")
204        python_bin_dir = self._install_dir / 'bin'
205        if not python_bin_dir.is_dir():
206            return
207
208        windows_suffixes = ('.exe', '.dll')
209        for f in python_bin_dir.iterdir():
210            if f.suffix not in windows_suffixes or f.is_symlink():
211                self._logger.info(f"Removing file: {f}")
212                f.unlink()
213                continue
214            self._strip_in_place(f)
215
216    def _remove_dir(self, dir_path: Path) -> None:
217        if dir_path.is_dir():
218            self._logger.info(f"Removing directory: {dir_path}")
219            shutil.rmtree(dir_path)
220
221    def _clean_share_dir(self) -> None:
222        self._logger.info("Cleaning share directory...")
223        share_dir = self._install_dir / 'share'
224        self._remove_dir(share_dir)
225
226    def _clean_lib_dir(self) -> None:
227        self._logger.info("Cleaning lib directory...")
228        python_lib_dir = self._install_dir / 'lib'
229        pkgconfig_dir = python_lib_dir / 'pkgconfig'
230        self._remove_dir(pkgconfig_dir)
231
232    def _remove_exclude(self) -> None:
233        self._logger.info("Removing excluded files and directories...")
234        exclude_dirs_tuple = (
235            f'config-{self._major_version}',
236            '__pycache__',
237            'idlelib',
238            'tkinter', 'turtledemo',
239            'test', 'tests'
240        )
241        exclude_files_tuple = (
242            'bdist_wininst.py',
243            'turtle.py',
244            '.whl',
245            '.pyc', '.pickle'
246        )
247
248        for root, dirs, files in os.walk(self._install_dir / 'lib'):
249            for item in dirs:
250                if item.startswith(exclude_dirs_tuple):
251                    self._logger.info(f"Removing directory: {os.path.join(root, item)}")
252                    shutil.rmtree(os.path.join(root, item))
253            for item in files:
254                if item.endswith(exclude_files_tuple):
255                    self._logger.info(f"Removing file: {os.path.join(root, item)}")
256                    os.remove(os.path.join(root, item))
257
258    def _copy_external_libs(self) -> None:
259        self._logger.info("Copying external libraries...")
260        # 定义源文件路径
261        _external_libs = [self._deps_dir / 'ffi' / 'bin' / 'libffi-8.dll', self._clang_toolchain_dir / self.build_config.MINGW_TRIPLE / 'bin' / 'libssp-0.dll']
262        # 定义目标目录
263        target_dir = self._install_dir / 'lib' / 'python3.11' / 'lib-dynload'
264        # 创建目标目录（如果不存在）
265        target_dir.mkdir(parents=True, exist_ok=True)
266
267        try:
268            for lib in _external_libs :
269                # 调用提取的方法拷贝 libffi-8.dll
270                self._copy_file_if_exists(lib, target_dir)
271        except Exception as e:
272            self._logger.error(f"Error copying external libraries: {e}")
273
274    def _copy_file_if_exists(self, src_path: Path, dest_dir: Path) -> None:
275        """
276        若源文件存在，则将其拷贝到目标目录，并记录相应日志；若不存在，则记录警告日志。
277
278        :param src_path: 源文件的路径
279        :param dest_dir: 目标目录的路径
280        """
281        if src_path.exists():
282            shutil.copy2(src_path, dest_dir)
283            self._logger.info(f"Copied {src_path} to {dest_dir}")
284        else:
285            self._logger.warning(f"{src_path} does not exist. Skipping.")
286
287    def _is_elf_file(self, file_path: Path) -> bool:
288        with open(file_path, 'rb') as f:
289            magic_numbers = f.read(4)
290            hex_magic_number = binascii.hexlify(magic_numbers).decode('utf-8')
291            return hex_magic_number == '7f454c46'
292
293    @property
294    def install_dir(self) -> str:
295        return str(self._install_dir)
296
297
298class MinGWPythonBuilder(PythonBuilder):
299    def __init__(self, build_config) -> None:
300        super().__init__(build_config)
301
302        self.target_platform = "x86_64-w64-mingw32"
303        self.patches = [f'cpython_mingw_v{self._version}.patch']
304        self._clang_toolchain_dir = Path(
305            os.path.join(self._prebuilts_path, 'mingw-w64', 'ohos', 'linux-x86_64', 'clang-mingw')).resolve()
306        self._mingw_install_dir = self._clang_toolchain_dir / build_config.MINGW_TRIPLE
307        self._build_dir = self._out_dir / 'python-windows-build'
308        self._install_dir = self._out_dir / 'python-windows-install'
309        self._deps_dir = self._out_dir / 'python-windows-deps'
310        # This file is used to detect whether patches are applied
311        self._patch_ignore_file = self._source_dir / 'mingw_ignorefile.txt'
312
313        for directory in (self._mingw_install_dir, self._source_dir):
314            if not directory.is_dir():
315                raise ValueError(f'No such directory "{directory}"')
316
317    def _extract_libffi(self):
318        """
319        定位 libffi-*.tar.gz 文件，清理输出目录后，将其直接解压到 out/libffi 目录。
320
321        Returns:
322            Path: 解压后的 libffi 内部目录的路径。
323
324        Raises:
325            FileNotFoundError: 若未找到 libffi-*.tar.gz 文件。
326            Exception: 若无法获取 libffi 压缩包的内部目录。
327        """
328        # 找到 libffi-*.tar.gz 包
329        libffi_tar_gz_files = glob.glob(str(self.repo_root / 'third_party' / 'libffi' / 'libffi-*.tar.gz'))
330        if not libffi_tar_gz_files:
331            self._logger.error("No libffi-*.tar.gz file found in third_party/libffi directory.")
332            raise FileNotFoundError("No libffi-*.tar.gz file found.")
333        libffi_tar_gz = libffi_tar_gz_files[0]
334
335        # 清理 out/libffi 目录
336        libffi_extract_dir = self._out_dir / 'libffi'
337        if libffi_extract_dir.exists():
338            self._logger.info(f"Cleaning existing libffi directory: {libffi_extract_dir}")
339            shutil.rmtree(libffi_extract_dir)
340        libffi_extract_dir.mkdir(parents=True)
341
342        # 直接解压 libffi-*.tar.gz 到 out/libffi 目录
343        with tarfile.open(libffi_tar_gz, 'r:gz') as tar:
344            tar.extractall(path=libffi_extract_dir)
345            # 获取解压后的目录名
346            members = tar.getmembers()
347            if members:
348                libffi_inner_dir = libffi_extract_dir / members[0].name
349            else:
350                self._logger.error("Failed to get inner directory of libffi tarball.")
351                raise Exception("Failed to get inner directory of libffi tarball.")
352        return libffi_inner_dir
353
354    @property
355    def _cflags(self) -> List[str]:
356        cflags = [
357            f'-target {self.target_platform}',
358            f'--sysroot={self._mingw_install_dir}',
359            f'-fstack-protector-strong',
360            f'-I{str(self._deps_dir / "ffi" / "include")}',
361            f'-nostdinc',
362            f'-I{str(self._mingw_install_dir / "include")}',
363            f'-I{str(self._clang_toolchain_dir / "lib" / "clang" / "15.0.4" / "include")}'
364        ]
365        return cflags
366
367    @property
368    def _ldflags(self) -> List[str]:
369        ldflags = [
370            f'--sysroot={self._mingw_install_dir}',
371            f'-rtlib=compiler-rt',
372            f'-target {self.target_platform}',
373            f'-lucrt',
374            f'-lucrtbase',
375            f'-fuse-ld=lld',
376            f'-L{str(self._deps_dir / "ffi" / "lib")}',
377        ]
378        return ldflags
379
380    @property
381    def _rcflags(self) -> List[str]:
382        return [f'-I{self._mingw_install_dir}/include']
383
384    def _deps_build(self) -> None:
385        self._logger.info("Starting MinGW dependency build process...")
386        # 调用提取的方法
387        libffi_inner_dir = self._extract_libffi()
388
389        env = os.environ.copy()
390        env.update({
391            'CC': "/bin/x86_64-w64-mingw32-gcc",
392            'CXX': "/bin/x86_64-w64-mingw32-g++",
393            'WINDRES': "/bin/x86_64-w64-mingw32-windres",
394            'AR': "/bin/x86_64-w64-mingw32-ar",
395            'READELF': "/bin/x86_64-w64-mingw32-readelf",
396            'LD': "/bin/x86_64-w64-mingw32-ld",
397            'DLLTOOL': "/bin/x86_64-w64-mingw32-dlltool",
398            'RANLIB': "/bin/x86_64-w64-mingw32-gcc-ranlib",
399            'STRIP': "/bin/x86_64-w64-mingw32-strip",
400            'CFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong",
401            'CXXFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong",
402            'LDFLAGS': "--sysroot=/usr/x86_64-w64-mingw32",
403            'RCFLAGS': "-I/usr/x86_64-w64-mingw32/include",
404            'CPPFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong"
405        })
406
407        configure_cmd = [
408            "./configure",
409            f"--prefix={self._deps_dir / 'ffi'}",
410            "--enable-shared",
411            "--build=x86_64-pc-linux-gnu",
412            "--host=x86_64-w64-mingw32",
413            "--disable-symvers",
414            "--disable-docs"
415        ]
416        run_command(configure_cmd, env=env, cwd=libffi_inner_dir)
417
418        # 执行 make -j16
419        make_cmd = ['make', '-j16']
420        run_command(make_cmd, env=env, cwd=libffi_inner_dir)
421
422        # 执行 make install
423        make_install_cmd = ['make', 'install']
424        run_command(make_install_cmd, env=env, cwd=libffi_inner_dir)
425
426    def _configure(self) -> None:
427        self._logger.info("Starting MinGW configuration...")
428        run_command(['autoreconf', '-vfi'], cwd=self._source_dir)
429        build_platform = subprocess.check_output(
430            ['./config.guess'], cwd=self._source_dir).decode().strip()
431        config_flags = [
432            f'--prefix={self._install_dir}',
433            f'--build={build_platform}',
434            f'--host={self.target_platform}',
435            f'--with-build-python={self._prebuilts_python_path}',
436            '--enable-shared',
437            '--without-ensurepip',
438            '--enable-loadable-sqlite-extensions',
439            '--disable-ipv6',
440            '--with-pydebug',
441            '--with-system-ffi'
442        ]
443        cmd = [str(self._source_dir / 'configure')] + config_flags
444        run_command(cmd, env=self._env, cwd=self._build_dir)
445
446    def prepare_for_package(self) -> None:
447        self._logger.info("Preparing MinGW build for packaging...")
448        self._clean_bin_dir()
449        self._clean_share_dir()
450        self._clean_lib_dir()
451        self._remove_exclude()
452        self._copy_external_libs()
453
454    def package(self) -> None:
455        self._logger.info("Packaging MinGW build...")
456        archive = self._out_dir / f'python-mingw-x86-{self._version}.tar.gz'
457        if archive.exists():
458            self._logger.info(f"Removing existing archive: {archive}")
459            archive.unlink()
460        cmd = [
461            'tar',
462            '-czf',
463            str(archive),
464            '--exclude=__pycache__',
465            '--transform',
466            f's,^,python/windows-x86/{self._lldb_py_version}/,',
467        ] + [f.name for f in self._install_dir.iterdir()]
468        run_command(cmd, cwd=self._install_dir)
469