• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from pathlib import Path
18import re
19import shutil
20import subprocess
21import sys
22from typing import List, Union
23from xml.dom import minidom
24
25
26# Use resolve to ensure consistent capitalization between runs of this script, which is important
27# for patching functions that insert a path only if it doesn't already exist.
28PYTHON_SRC = Path(__file__).parent.parent.resolve()
29TOP = PYTHON_SRC.parent.parent.parent
30
31
32def create_new_dir(path: Path) -> None:
33    if path.exists():
34        shutil.rmtree(path)
35    path.mkdir(parents=True)
36
37
38def run_cmd(args: List[Union[str, Path]], cwd: Path) -> None:
39    print(f'cd {cwd}')
40    str_args = [str(arg) for arg in args]
41    print(subprocess.list2cmdline(str_args))
42    subprocess.run(str_args, cwd=cwd, check=True)
43
44
45def read_xml_file(path: Path) -> minidom.Element:
46    doc = minidom.parse(str(path))
47    return doc.documentElement
48
49
50def write_xml_file(root: minidom.Element, path: Path) -> None:
51    with open(path, 'w', encoding='utf8') as out:
52        out.write('<?xml version="1.0" encoding="utf-8"?>\n')
53        root.writexml(out)
54
55
56def get_text_element(root: minidom.Element, tag: str) -> str:
57    (node,) = root.getElementsByTagName(tag)
58    (node,) = node.childNodes
59    assert node.nodeType == root.TEXT_NODE
60    assert isinstance(node.data, str)
61    return node.data
62
63
64def set_text_element(root: minidom.Element, tag: str, new_text: str) -> None:
65    (node,) = root.getElementsByTagName(tag)
66    (node,) = node.childNodes
67    assert node.nodeType == root.TEXT_NODE
68    node.data = new_text
69
70
71def patch_python_for_licenses():
72    # Python already handles bzip2 and libffi itself.
73    notice_files = [
74        TOP / 'external/zlib/LICENSE',
75        TOP / 'toolchain/xz/COPYING',
76    ]
77
78    xml_path = PYTHON_SRC / 'PCbuild/regen.targets'
79    proj = read_xml_file(xml_path)
80
81    # Pick the unconditional <_LicenseSources> element and add extra notices to the end.
82    elements = proj.getElementsByTagName('_LicenseSources')
83    (element,) = [e for e in elements if not e.hasAttribute('Condition')]
84    includes = element.getAttribute('Include').split(';')
85    for notice in notice_files:
86        if str(notice) not in includes:
87            includes.append(str(notice))
88    element.setAttribute('Include', ';'.join(includes))
89
90    write_xml_file(proj, xml_path)
91
92
93def remove_modules_from_pcbuild_proj():
94    modules_to_remove = [
95        '_sqlite3',
96    ]
97
98    xml_path = PYTHON_SRC / 'PCbuild/pcbuild.proj'
99    proj = read_xml_file(xml_path)
100    for tag in ('ExternalModules', 'ExtensionModules'):
101        for element in proj.getElementsByTagName(tag):
102            deps = element.getAttribute('Include').split(';')
103            for unwanted in modules_to_remove:
104                if unwanted in deps:
105                    deps.remove(unwanted)
106                    element.setAttribute('Include', ';'.join(deps))
107    write_xml_file(proj, xml_path)
108
109
110def build_using_cmake(out: Path, src: Path) -> None:
111    create_new_dir(out)
112    cmake = TOP / 'prebuilts/cmake/windows-x86/bin/cmake.exe'
113    run_cmd([cmake, src, '-G', 'Visual Studio 15 2017 Win64'], cwd=out)
114    run_cmd([cmake, '--build', '.', '--config', 'Release'], cwd=out)
115
116
117def patch_libffi_props() -> None:
118    """The libffi.props file uses libffi-N.lib and libffi-N.dll. (At time of writing, N is 7, but
119    upstream Python uses 8 instead.) The CMake-based build of libffi produces unsuffixed files, so
120    fix libffi.props to match.
121    """
122    path = PYTHON_SRC / 'PCbuild/libffi.props'
123    content = path.read_text(encoding='utf8')
124    content = re.sub(r'libffi-\d+\.lib', 'libffi.lib', content)
125    content = re.sub(r'libffi-\d+\.dll', 'libffi.dll', content)
126    path.write_text(content, encoding='utf8')
127
128
129def patch_pythoncore_for_zlib() -> None:
130    """pythoncore.vcxproj builds zlib into itself by listing individual zlib C files. AOSP uses
131    Chromium's zlib fork, which has a different set of C files and defines. Switch to AOSP zlib:
132     - Build a static library using CMake: libz.lib, zconf.h, zlib.h
133     - Strip ClCompile/ClInclude elements from the project file that point to $(zlibDir).
134     - Add a dependency on the static library.
135    """
136
137    xml_path = PYTHON_SRC / 'PCbuild/pythoncore.vcxproj'
138    proj = read_xml_file(xml_path)
139
140    # Strip ClCompile/ClInclude that point into the zlib directory.
141    for tag in ('ClCompile', 'ClInclude'):
142        for element in proj.getElementsByTagName(tag):
143            if element.getAttribute('Include').startswith('$(zlibDir)'):
144                element.parentNode.removeChild(element)
145
146    # Add a dependency on the static zlib archive.
147    deps = get_text_element(proj, 'AdditionalDependencies').split(';')
148    libz_path = str(TOP / 'out/zlib/Release/libz.lib')
149    if libz_path not in deps:
150        deps.insert(0, libz_path)
151        set_text_element(proj, 'AdditionalDependencies', ';'.join(deps))
152
153    write_xml_file(proj, xml_path)
154
155
156def main() -> None:
157    # Point the Python MSBuild project at the paths where repo/Kokoro would put the various
158    # dependencies. The existing python.props uses trailing slashes in the path, and some (but not
159    # all) uses of these variables expect the trailing slash.
160    xml_path = PYTHON_SRC / 'PCbuild/python.props'
161    root = read_xml_file(xml_path)
162    set_text_element(root, 'bz2Dir', str(TOP / 'external/bzip2') + '\\')
163    set_text_element(root, 'libffiDir', str(TOP / 'external/libffi') + '\\') # Provides LICENSE
164    set_text_element(root, 'libffiIncludeDir', str(TOP / 'out/libffi/dist/include') + '\\') # headers
165    set_text_element(root, 'libffiOutDir', str(TOP / 'out/libffi/Release') + '\\') # dll+lib
166    set_text_element(root, 'lzmaDir', str(TOP / 'toolchain/xz') + '\\')
167    set_text_element(root, 'zlibDir', str(TOP / 'out/zlib/dist/include') + '\\')
168    write_xml_file(root, xml_path)
169
170    # liblzma.vcxproj adds $(lzmaDir)windows to the include path for config.h, but AOSP has a newer
171    # version of xz that moves config.h into a subdir like vs2017 or vs2019. See this upstream
172    # commit [1]. Copy the file into the place Python currently expects. (This can go away if Python
173    # updates its xz dependency.)
174    #
175    # [1] https://git.tukaani.org/?p=xz.git;a=commit;h=82388980187b0e3794d187762054200bbdcc9a53
176    xz = TOP / 'toolchain/xz'
177    shutil.copy2(xz / 'windows/vs2017/config.h',
178                 xz / 'windows/config.h')
179
180    patch_python_for_licenses()
181    remove_modules_from_pcbuild_proj()
182    build_using_cmake(TOP / 'out/libffi', TOP / 'external/libffi')
183    build_using_cmake(TOP / 'out/zlib', TOP / 'external/zlib')
184    patch_libffi_props()
185    patch_pythoncore_for_zlib()
186
187
188if __name__ == '__main__':
189    main()
190