1#!/bin/sh 2# 3# Copyright (C) 2012 The Android Open Source Project 4# Copyright (C) 2012 Ray Donnelly <mingw.android at gmail.com> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18# Rebuild the host Python binaries from sources. 19# 20 21# include common function and variable definitions 22NDK_BUILDTOOLS_PATH="$(dirname $0)" 23. "$NDK_BUILDTOOLS_PATH/prebuilt-common.sh" 24. "$NDK_BUILDTOOLS_PATH/common-build-host-funcs.sh" 25 26PROGRAM_PARAMETERS="" 27PROGRAM_DESCRIPTION="\ 28This program is used to rebuild one or more Python client programs from 29sources. To use it, you will need a working set of toolchain sources, like 30those downloaded with download-toolchain-sources.sh, then pass the 31corresponding directory with the --toolchain-src-dir=<path> option. 32 33By default, the script rebuilds Python for you host system [$HOST_TAG], 34but you can use --systems=<tag1>,<tag2>,.. to ask binaries that can run on 35several distinct systems. Each <tag> value in the list can be one of the 36following: 37 38 linux-x86 39 linux-x86_64 40 windows 41 windows-x86 (equivalent to 'windows') 42 windows-x86_64 43 darwin-x86 44 darwin-x86_64 45 46For example, here's how to rebuild Python 2.7.3 on Linux 47for six different systems: 48 49 $PROGNAME --build-dir=/path/to/toolchain/src \ 50 --python-version=2.7.3 \ 51 --systems=linux-x86,linux-x86_64,windows,windows-x86_64,darwin-x86,darwin-x86_64" 52 53TOOLCHAIN_SRC_DIR= 54register_var_option "--toolchain-src-dir=<path>" TOOLCHAIN_SRC_DIR "Select toolchain source directory" 55 56PYTHON_VERSION=$DEFAULT_PYTHON_VERSION 57register_var_option "--python-version=<version>" PYTHON_VERSION "Select Python version." 58 59NDK_DIR=$ANDROID_NDK_ROOT 60register_var_option "--ndk-dir=<path>" NDK_DIR "Select NDK install directory." 61 62PACKAGE_DIR= 63register_var_option "--package-dir=<path>" PACKAGE_DIR "Package prebuilt tarballs into directory." 64 65BUILD_DIR= 66register_var_option "--build-dir=<path>" BUILD_DIR "Build Python into directory" 67 68bh_register_options 69 70register_jobs_option 71 72extract_parameters "$@" 73 74if [ -n "$PARAMETERS" ]; then 75 panic "This script doesn't take parameters, only options. See --help" 76fi 77 78if [ -z "$TOOLCHAIN_SRC_DIR" ]; then 79 panic "Please use --toolchain-src-dir=<path> to select toolchain source directory." 80fi 81 82BH_HOST_SYSTEMS=$(commas_to_spaces $BH_HOST_SYSTEMS) 83 84# Python needs to execute itself during its build process, so must build the build 85# Python first. It should also be an error if not asked to build for build machine. 86BH_HOST_SYSTEMS=$(bh_sort_systems_build_first "$BH_HOST_SYSTEMS") 87 88# Make sure that the the user asked for the build OS's Python to be built. 89# and that the above sort command worked correctly. 90if [ ! "$(bh_list_contains $BH_BUILD_TAG $BH_HOST_SYSTEMS)" = "first" ] ; then 91 panic "Cross building Python requires building for the build OS!" 92fi 93 94download_package () 95{ 96 # Assume the packages are already downloaded under $ARCHIVE_DIR 97 local PKG_URL=$1 98 local PKG_NAME=$(basename $PKG_URL) 99 100 case $PKG_NAME in 101 *.tar.bz2) 102 PKG_BASENAME=${PKG_NAME%%.tar.bz2} 103 ;; 104 *.tar.gz) 105 PKG_BASENAME=${PKG_NAME%%.tar.gz} 106 ;; 107 *) 108 panic "Unknown archive type: $PKG_NAME" 109 esac 110 111 if [ ! -f "$ARCHIVE_DIR/$PKG_NAME" ]; then 112 log "Downloading $PKG_URL..." 113 (cd $ARCHIVE_DIR && run curl -L -o "$PKG_NAME" "$PKG_URL") 114 fail_panic "Can't download '$PKG_URL'" 115 fi 116 117 if [ ! -d "$SRC_DIR/$PKG_BASENAME" ]; then 118 log "Uncompressing $PKG_URL into $SRC_DIR" 119 case $PKG_NAME in 120 *.tar.bz2) 121 run tar xjf $ARCHIVE_DIR/$PKG_NAME -C $SRC_DIR 122 ;; 123 *.tar.gz) 124 run tar xzf $ARCHIVE_DIR/$PKG_NAME -C $SRC_DIR 125 ;; 126 *) 127 panic "Unknown archive type: $PKG_NAME" 128 ;; 129 esac 130 fail_panic "Can't uncompress $ARCHIVE_DIR/$PKG_NAME" 131 fi 132} 133 134if [ -z "$BUILD_DIR" ] ; then 135 BUILD_DIR=/tmp/ndk-$USER/buildhost 136fi 137 138bh_setup_build_dir $BUILD_DIR 139 140if [ "$BH_BUILD_MODE" = "debug" ] ; then 141 PYDEBUG="-with-pydebug" 142fi 143 144# Sanity check that we have the right compilers for all hosts 145for SYSTEM in $BH_HOST_SYSTEMS; do 146 bh_setup_build_for_host $SYSTEM 147done 148 149TEMP_DIR=$BUILD_DIR/tmp 150# Download and unpack source packages from official sites 151ARCHIVE_DIR=$TEMP_DIR/archive 152STAMP_DIR=$TEMP_DIR/timestamps 153BUILD_DIR=$TEMP_DIR/build-$HOST_TAG 154 155mkdir -p $BUILD_DIR 156 157PROGDIR=`dirname $0` 158PROGDIR=$(cd $PROGDIR && pwd) 159 160# Sanity check for all Python versions. 161for VERSION in $(commas_to_spaces $PYTHON_VERSION); do 162 PYTHON_SRCDIR=$TOOLCHAIN_SRC_DIR/python/Python-$VERSION 163 if [ ! -d "$PYTHON_SRCDIR" ]; then 164 panic "Missing source directory: $PYTHON_SRCDIR" 165 fi 166done 167 168# Return the build install directory of a given Python version 169# $1: host system tag 170# $2: python version 171# The suffix of this has to match python_ndk_install_dir 172# as I package them from the build folder, substituting 173# the end part of python_build_install_dir matching python_ndk_install_dir 174# with nothing. 175# Needs to match with 176# python_build_install_dir () in build-host-gdb.sh 177python_build_install_dir () 178{ 179 echo "$BH_BUILD_DIR/install/prebuilt/$1" 180} 181 182# $1: host system tag 183# $2: python version 184python_ndk_package_name () 185{ 186 echo "python-$2" 187} 188 189 190# Same as python_build_install_dir, but for the final NDK installation 191# directory. Relative to $NDK_DIR. 192python_ndk_install_dir () 193{ 194 echo "prebuilt/$1" 195} 196 197arch_to_qemu_arch () 198{ 199 case $1 in 200 x86) 201 echo i386 202 ;; 203 *) 204 echo $1 205 ;; 206 esac 207} 208 209# $1: host system tag 210# $2: python version 211build_host_python () 212{ 213 local SRCDIR=$TOOLCHAIN_SRC_DIR/python/Python-$2 214 local BUILDDIR=$BH_BUILD_DIR/build-python-$1-$2 215 local INSTALLDIR=$(python_build_install_dir $1 $2) 216 local ARGS TEXT 217 218 if [ ! -f "$SRCDIR/configure" ]; then 219 panic "Missing configure script in $SRCDIR" 220 fi 221 222 ARGS=" --prefix=$INSTALLDIR" 223 224 # Python considers it cross compiling if --host is passed 225 # and that then requires that a CONFIG_SITE file is used. 226 # This is not necessary if it's only the arch that differs. 227 if [ ! $BH_HOST_CONFIG = $BH_BUILD_CONFIG ] ; then 228 ARGS=$ARGS" --build=$BH_BUILD_CONFIG" 229 fi 230 ARGS=$ARGS" --host=$BH_HOST_CONFIG" 231 ARGS=$ARGS" $PYDEBUG" 232 ARGS=$ARGS" --disable-ipv6" 233 234 mkdir -p "$BUILDDIR" && rm -rf "$BUILDDIR"/* 235 cd "$BUILDDIR" && 236 bh_setup_host_env 237 238 CFG_SITE= 239 # Need to add -L$HOST_STATIC_LIBDIR to LDSHARED if need 240 # any static host libs. 241 export LDSHARED="$CC -shared " 242 if [ ! $BH_HOST_TAG = $BH_BUILD_TAG ]; then 243 244 # Cross compiling. 245 CFG_SITE=$BUILDDIR/config.site 246 247 # Ideally would remove all of these configury hacks by 248 # patching the issues. 249 250 if [ $1 = darwin-x86 -o $1 = darwin-x86_64 ]; then 251 echo "ac_cv_file__dev_ptmx=no" > $CFG_SITE 252 echo "ac_cv_file__dev_ptc=no" >> $CFG_SITE 253 echo "ac_cv_have_long_long_format=yes" >> $CFG_SITE 254 if [ $1 = darwin-x86 ] ; then 255 echo "ac_osx_32bit=yes" >> $CFG_SITE 256 elif [ $1 = darwin-x86_64 ] ; then 257 echo "ac_osx_32bit=no" >> $CFG_SITE 258 fi 259 echo "ac_cv_have_sendfile=no" >> $CFG_SITE 260 # I could change AC_MSG_CHECKING(LDSHARED) in configure.ac 261 # to check $host instead of $ac_sys_system/$ac_sys_release 262 # but it handles loads of platforms 263 # and I can only test on three, so instead... 264 export LDSHARED="$CC -bundle -undefined dynamic_lookup" 265 elif [ $1 = windows-x86 -o $1 = windows-x86_64 ]; then 266 echo "ac_cv_file__dev_ptmx=no" > $CFG_SITE 267 echo "ac_cv_file__dev_ptc=no" >> $CFG_SITE 268 CFLAGS=$CFLAGS" -D__USE_MINGW_ANSI_STDIO=1" 269 CXXFLAGS=$CXXFLAGS" -D__USE_MINGW_ANSI_STDIO=1" 270 elif [ $1 = linux-x86 -o $1 = linux-x86_64 ]; then 271 echo "ac_cv_file__dev_ptmx=yes" > $CFG_SITE 272 echo "ac_cv_file__dev_ptc=no" >> $CFG_SITE 273 echo "ac_cv_have_long_long_format=yes" >> $CFG_SITE 274 echo "ac_cv_pthread_system_supported=yes" >> $CFG_SITE 275 echo "ac_cv_working_tzset=yes" >> $CFG_SITE 276 echo "ac_cv_little_endian_double=yes" >> $CFG_SITE 277 fi 278 279 if [ $BH_HOST_OS = $BH_BUILD_OS ]; then 280 # Only cross compiling from arch perspective. 281 # qemu causes failures as cross-compilation is not detected 282 # if a test executable can be run successfully, so we test 283 # for qemu-${BH_HOST_ARCH} and qemu-${BH_HOST_ARCH}-static 284 # and panic if either are found. 285 QEMU_HOST_ARCH=$(arch_to_qemu_arch $BH_HOST_ARCH) 286 if [ ! -z "$(which qemu-$QEMU_HOST_ARCH 2>/dev/null)" -o \ 287 ! -z "$(which qemu-$QEMU_HOST_ARCH-static 2>/dev/null)" ] ; then 288 panic "Installed qemu(s) ($(which qemu-$QEMU_HOST_ARCH 2>/dev/null) $(which qemu-$QEMU_HOST_ARCH-static 2>/dev/null))" \ 289 "will prevent this build from working." 290 fi 291 fi 292 fi 293 294 TEXT="$(bh_host_text) python-$BH_HOST_CONFIG:" 295 296 touch $SRCDIR/Include/graminit.h 297 touch $SRCDIR/Python/graminit.c 298 echo "" > $SRCDIR/Parser/pgen.stamp 299 300 dump "$TEXT Building" 301 export CONFIG_SITE=$CFG_SITE && 302 run2 "$SRCDIR"/configure $ARGS && 303 run2 make -j$NUM_JOBS && 304 run2 make -j$NUM_JOBS install 305} 306 307need_build_host_python () 308{ 309 bh_stamps_do host-python-$1-$2 build_host_python $1 $2 310} 311 312# Install host Python binaries and support files to the NDK install dir. 313# $1: host tag 314# $2: python version 315install_host_python () 316{ 317 local SRCDIR="$(python_build_install_dir $1 $2)" 318 local DSTDIR="$NDK_DIR/$(python_ndk_install_dir $1 $2)" 319 320 need_build_host_python $1 $2 321 322 dump "$(bh_host_text) python-$BH_HOST_ARCH-$2: Installing" 323 run copy_directory "$SRCDIR/bin" "$DSTDIR/bin" 324 run copy_directory "$SRCDIR/lib" "$DSTDIR/lib" 325 run copy_directory "$SRCDIR/share" "$DSTDIR/share" 326 run copy_directory "$SRCDIR/include" "$DSTDIR/include" 327} 328 329need_install_host_python () 330{ 331 local SRCDIR="$(python_build_install_dir $1 $2)" 332 333 bh_stamps_do install-host-python-$1-$2 install_host_python $1 $2 334 335 # make sharedmods (setup.py) needs to use the build machine's Python 336 # for the other hosts to build correctly. 337 if [ $BH_BUILD_TAG = $BH_HOST_TAG ]; then 338 export PATH=$SRCDIR/bin:$PATH 339 fi 340} 341 342# Package host Python binaries into a tarball 343# $1: host tag 344# $2: python version 345package_host_python () 346{ 347 local BLDDIR="$(python_build_install_dir $1 $2)" 348 local SRCDIR="$(python_ndk_install_dir $1 $2)" 349 # This is similar to BLDDIR=${BLDDIR%%$SRCDIR} 350 BLDDIR=$(echo "$BLDDIR" | sed "s/$(echo "$SRCDIR" | sed -e 's/\\/\\\\/g' -e 's/\//\\\//g' -e 's/&/\\\&/g')//g") 351 local PACKAGENAME=$(python_ndk_package_name $1 $2)-$1.tar.bz2 352 local PACKAGE="$PACKAGE_DIR/$PACKAGENAME" 353 354 need_install_host_python $1 $2 355 356 dump "$(bh_host_text) $PACKAGENAME: Packaging" 357 run pack_archive "$PACKAGE" "$BLDDIR" "$SRCDIR" 358} 359 360PYTHON_VERSION=$(commas_to_spaces $PYTHON_VERSION) 361ARCHS=$(commas_to_spaces $ARCHS) 362 363# Let's build this 364for SYSTEM in $BH_HOST_SYSTEMS; do 365 bh_setup_build_for_host $SYSTEM 366 for VERSION in $PYTHON_VERSION; do 367 need_install_host_python $SYSTEM $VERSION 368 done 369done 370 371if [ "$PACKAGE_DIR" ]; then 372 for SYSTEM in $BH_HOST_SYSTEMS; do 373 bh_setup_build_for_host $SYSTEM 374 for VERSION in $PYTHON_VERSION; do 375 package_host_python $SYSTEM $VERSION 376 done 377 done 378fi 379