1#!/bin/bash 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16set -ex 17 18cd "$(dirname "$0")/../../.." 19 20export GRPC_PYTHON_BUILD_WITH_CYTHON=1 21export PYTHON=${PYTHON:-python} 22export AUDITWHEEL=${AUDITWHEEL:-auditwheel} 23 24# activate ccache if desired 25# shellcheck disable=SC1091 26source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc 27 28# Needed for building binary distribution wheels -- bdist_wheel 29"${PYTHON}" -m pip install --upgrade pip 30# Ping to a single version to make sure we're building the same artifacts 31"${PYTHON}" -m pip install setuptools==69.5.1 wheel==0.43.0 32 33if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ] 34then 35 # Install Cython to avoid source wheel build failure. 36 # This only needs to be done when not running under docker (=on MacOS) 37 # since the docker images used for building python wheels 38 # already have a new-enough version of cython pre-installed. 39 # Any installation step is a potential source of breakages, 40 # so we are trying to perform as few download-and-install operations 41 # as possible. 42 "${PYTHON}" -m pip install --upgrade 'cython<4.0.0rc1' 43fi 44 45# Allow build_ext to build C/C++ files in parallel 46# by enabling a monkeypatch. It speeds up the build a lot. 47# Use externally provided GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS value if set. 48export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=${GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS:-2} 49 50mkdir -p "${ARTIFACTS_OUT}" 51ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}" 52 53# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image. 54if [ "$AUDITWHEEL_ARCH" == "aarch64" ] 55then 56 # when crosscompiling for aarch64, --plat-name needs to be set explicitly 57 # to end up with correctly named wheel file 58 # the value should be manylinuxABC_ARCH and dockcross docker image 59 # conveniently provides the value in the AUDITWHEEL_PLAT env 60 WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT" 61 62 # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix 63 GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')" 64 export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX 65 66 # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations 67 export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-aarch64" 68fi 69 70# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image. 71if [ "$AUDITWHEEL_ARCH" == "armv7l" ] 72then 73 # when crosscompiling for arm, --plat-name needs to be set explicitly 74 # to end up with correctly named wheel file 75 # our dockcross-based docker image onveniently provides the value in the AUDITWHEEL_PLAT env 76 WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT" 77 78 # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix 79 GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-arm-linux-gnueabihf.so"))')" 80 export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX 81 82 # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations 83 export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-arm" 84fi 85 86ancillary_package_dir=( 87 "src/python/grpcio_admin/" 88 "src/python/grpcio_channelz/" 89 "src/python/grpcio_csds/" 90 "src/python/grpcio_health_checking/" 91 "src/python/grpcio_reflection/" 92 "src/python/grpcio_status/" 93 "src/python/grpcio_testing/" 94 "src/python/grpcio_observability/" 95 "src/python/grpcio_csm_observability/" 96) 97 98# Copy license to ancillary package directories so it will be distributed. 99for directory in "${ancillary_package_dir[@]}"; do 100 cp "LICENSE" "${directory}" 101done 102 103# Build the source distribution first because MANIFEST.in cannot override 104# exclusion of built shared objects among package resources (for some 105# inexplicable reason). 106${SETARCH_CMD} "${PYTHON}" setup.py sdist 107 108# Wheel has a bug where directories don't get excluded. 109# https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory 110# shellcheck disable=SC2086 111${SETARCH_CMD} "${PYTHON}" setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG 112 113GRPCIO_STRIP_TEMPDIR=$(mktemp -d) 114GRPCIO_TAR_GZ_LIST=( dist/grpcio-*.tar.gz ) 115GRPCIO_TAR_GZ=${GRPCIO_TAR_GZ_LIST[0]} 116GRPCIO_STRIPPED_TAR_GZ=$(mktemp -t "TAR_GZ_XXXXXXXXXX") 117 118clean_non_source_files() { 119( cd "$1" 120 find . -type f \ 121 | grep -v '\.c$' | grep -v '\.cc$' | grep -v '\.cpp$' \ 122 | grep -v '\.h$' | grep -v '\.hh$' | grep -v '\.inc$' \ 123 | grep -v '\.s$' | grep -v '\.py$' | grep -v '\.hpp$' \ 124 | grep -v '\.S$' | grep -v '\.asm$' \ 125 | while read -r file; do 126 rm -f "$file" || true 127 done 128 find . -type d -empty -delete 129) 130} 131 132tar xzf "${GRPCIO_TAR_GZ}" -C "${GRPCIO_STRIP_TEMPDIR}" 133( cd "${GRPCIO_STRIP_TEMPDIR}" 134 find . -type d -name .git -exec rm -fr {} \; || true 135 for dir in */third_party/*; do 136 clean_non_source_files "${dir}" || true 137 done 138 tar czf "${GRPCIO_STRIPPED_TAR_GZ}" -- * 139 chmod ugo+r "${GRPCIO_STRIPPED_TAR_GZ}" 140) 141mv "${GRPCIO_STRIPPED_TAR_GZ}" "${GRPCIO_TAR_GZ}" 142 143# Build gRPC tools package distribution 144"${PYTHON}" tools/distrib/python/make_grpcio_tools.py 145 146# Build gRPC tools package source distribution 147${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py sdist 148 149# Build gRPC tools package binary distribution 150# shellcheck disable=SC2086 151${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG 152 153if [ "$GRPC_BUILD_MAC" == "" ]; then 154 "${PYTHON}" src/python/grpcio_observability/make_grpcio_observability.py 155 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py sdist 156 # shellcheck disable=SC2086 157 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG 158fi 159 160 161# run twine check before auditwheel, because auditwheel puts the repaired wheels into 162# the artifacts output dir. 163if [ "$GRPC_SKIP_TWINE_CHECK" == "" ] 164then 165 # Install virtualenv if it isn't already available. 166 # TODO(jtattermusch): cleanup the virtualenv version fallback logic. 167 "${PYTHON}" -m pip install virtualenv 168 "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==20.0.23 && "${PYTHON}" -m virtualenv venv; } 169 # Ensure the generated artifacts are valid using "twine check" 170 # pinning twine's dependency package `cryptography` version to 3.3.2 (last version without Rust dependency) 171 venv/bin/python -m pip install "cryptography==3.3.2" "twine==5.0.0" "readme_renderer<40.0" 172 venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/* 173 if [ "$GRPC_BUILD_MAC" == "" ]; then 174 venv/bin/python -m twine check src/python/grpcio_observability/dist/* 175 fi 176 rm -rf venv/ 177fi 178 179assert_is_universal_wheel() { 180 WHL="$1" 181 TMPDIR=$(mktemp -d) 182 unzip "$WHL" -d "$TMPDIR" 183 SO=$(find "$TMPDIR" -name '*.so' | head -n1) 184 if ! file "$SO" | grep "Mach-O universal binary with 2 architectures"; then 185 echo "$WHL is not universal2. Found the following:" >/dev/stderr 186 file "$SO" >/dev/stderr 187 exit 1 188 fi 189} 190 191fix_faulty_universal2_wheel() { 192 WHL="$1" 193 assert_is_universal_wheel "$WHL" 194 if echo "$WHL" | grep "x86_64"; then 195 UPDATED_NAME="${WHL//x86_64/universal2}" 196 mv "$WHL" "$UPDATED_NAME" 197 fi 198} 199 200# This is necessary due to https://github.com/pypa/wheel/issues/406. 201# wheel incorrectly generates a universal2 artifact that only contains 202# x86_64 libraries. 203if [ "$GRPC_BUILD_MAC" != "" ]; then 204 for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do 205 fix_faulty_universal2_wheel "$WHEEL" 206 done 207fi 208 209 210if [ "$GRPC_RUN_AUDITWHEEL_REPAIR" != "" ] 211then 212 for wheel in dist/*.whl; do 213 "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" 214 "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" 215 rm "$wheel" 216 done 217 for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do 218 "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" 219 "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" 220 rm "$wheel" 221 done 222else 223 cp -r dist/*.whl "$ARTIFACT_DIR" 224 cp -r tools/distrib/python/grpcio_tools/dist/*.whl "$ARTIFACT_DIR" 225fi 226 227# grpcio and grpcio-tools have already been copied to artifact_dir 228# by "auditwheel repair", now copy the .tar.gz source archives as well. 229cp -r dist/*.tar.gz "$ARTIFACT_DIR" 230cp -r tools/distrib/python/grpcio_tools/dist/*.tar.gz "$ARTIFACT_DIR" 231 232 233if [ "$GRPC_BUILD_MAC" == "" ]; then 234 if [ "$GRPC_RUN_AUDITWHEEL_REPAIR" != "" ] 235 then 236 for wheel in src/python/grpcio_observability/dist/*.whl; do 237 "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" 238 "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" 239 rm "$wheel" 240 done 241 else 242 cp -r src/python/grpcio_observability/dist/*.whl "$ARTIFACT_DIR" 243 fi 244 cp -r src/python/grpcio_observability/dist/*.tar.gz "$ARTIFACT_DIR" 245 246 # Build grpcio_csm_observability distribution 247 if [ "$GRPC_BUILD_MAC" == "" ]; then 248 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_csm_observability/setup.py \ 249 sdist bdist_wheel 250 cp -r src/python/grpcio_csm_observability/dist/* "$ARTIFACT_DIR" 251 fi 252fi 253 254# We need to use the built grpcio-tools/grpcio to compile the health proto 255# Wheels are not supported by setup_requires/dependency_links, so we 256# manually install the dependency. Note we should only do this if we 257# are in a docker image or in a virtualenv. 258if [ "$GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS" != "" ] 259then 260 "${PYTHON}" -m pip install -rrequirements.txt 261 262 if [ "$("$PYTHON" -c "import sys; print(sys.version_info[0])")" == "2" ] 263 then 264 # shellcheck disable=SC2261 265 "${PYTHON}" -m pip install futures>=2.2.0 enum34>=1.0.4 266 fi 267 268 "${PYTHON}" -m pip install grpcio --no-index --find-links "file://$ARTIFACT_DIR/" 269 "${PYTHON}" -m pip install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/" 270 271 # Note(lidiz) setuptools's "sdist" command creates a source tarball, which 272 # demands an extra step of building the wheel. The building step is merely ran 273 # through setup.py, but we can optimize it with "bdist_wheel" command, which 274 # skips the wheel building step. 275 276 # Build xds_protos source distribution 277 # build.py is invoked as part of generate_projects. 278 ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/xds_protos/setup.py \ 279 sdist bdist_wheel install 280 cp -r tools/distrib/python/xds_protos/dist/* "$ARTIFACT_DIR" 281 282 # Build grpcio_testing source distribution 283 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_testing/setup.py preprocess \ 284 sdist bdist_wheel 285 cp -r src/python/grpcio_testing/dist/* "$ARTIFACT_DIR" 286 287 # Build grpcio_channelz source distribution 288 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_channelz/setup.py \ 289 preprocess build_package_protos sdist bdist_wheel 290 cp -r src/python/grpcio_channelz/dist/* "$ARTIFACT_DIR" 291 292 # Build grpcio_health_checking source distribution 293 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_health_checking/setup.py \ 294 preprocess build_package_protos sdist bdist_wheel 295 cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR" 296 297 # Build grpcio_reflection source distribution 298 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_reflection/setup.py \ 299 preprocess build_package_protos sdist bdist_wheel 300 cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR" 301 302 # Build grpcio_status source distribution 303 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_status/setup.py \ 304 preprocess sdist bdist_wheel 305 cp -r src/python/grpcio_status/dist/* "$ARTIFACT_DIR" 306 307 # Install xds-protos as a dependency of grpcio-csds 308 "${PYTHON}" -m pip install xds-protos --no-index --find-links "file://$ARTIFACT_DIR/" 309 310 # Build grpcio_csds source distribution 311 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_csds/setup.py \ 312 sdist bdist_wheel 313 cp -r src/python/grpcio_csds/dist/* "$ARTIFACT_DIR" 314 315 # Build grpcio_admin source distribution and it needs the cutting-edge version 316 # of Channelz and CSDS to be installed. 317 "${PYTHON}" -m pip install grpcio-channelz --no-index --find-links "file://$ARTIFACT_DIR/" 318 "${PYTHON}" -m pip install grpcio-csds --no-index --find-links "file://$ARTIFACT_DIR/" 319 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_admin/setup.py \ 320 sdist bdist_wheel 321 cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR" 322 323fi 324