1#!/bin/bash -ex 2 3# Copyright 2020 Google Inc. All rights reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17: "${OUT_DIR:?Must set OUT_DIR}" 18TOP=$(pwd) 19 20UNAME="$(uname)" 21case "$UNAME" in 22Linux) 23 OS='linux' 24 ;; 25Darwin) 26 OS='darwin' 27 ;; 28*) 29 echo "Unrecognized OS $UNAME" 30 exit 1 31 ;; 32esac 33 34UNAME_ARCH=$(uname -m) 35case "$UNAME_ARCH" in 36x86_64) 37 ARCH='x86' 38 ;; 39aarch64) 40 ARCH="arm64" 41 ;; 42*) 43 echo "Unrecognized architecture $UNAME_ARCH" 44 exit 1 45 ;; 46esac 47 48build_soong=1 49build_asan=1 50build_native=1 51build_java=1 52[[ ! -d ${TOP}/toolchain/go ]] || build_go=1 53 54use_musl=false 55clean=true 56while getopts ":-:" opt; do 57 case "$opt" in 58 -) 59 case "${OPTARG}" in 60 resume) clean= ;; 61 musl) use_musl=true; build_arm64_cross_musl=1 ;; 62 skip-go) unset build_go ;; 63 skip-soong) unset build_soong ;; 64 skip-soong-tests) skip_soong_tests=--skip-soong-tests ;; 65 skip-asan) unset build_asan ;; 66 *) echo "Unknown option --${OPTARG}"; exit 1 ;; 67 esac;; 68 *) echo "'${opt}' '${OPTARG}'" 69 esac 70done 71 72primary_arch="" 73if [[ ${ARCH} = x86 ]]; then 74 primary_arch="\"HostArch\":\"x86_64\"," 75elif [[ ${ARCH} = arm64 ]]; then 76 primary_arch="\"HostArch\":\"arm64\"," 77 use_musl=true 78 unset build_native 79 unset build_java 80 unset build_asan 81 unset build_arm64_cross_musl 82 skip_soong_tests=--skip-soong-tests 83fi 84 85secondary_arch="" 86if [[ ${OS} = linux && ${ARCH} = x86 ]]; then 87 secondary_arch="\"HostSecondaryArch\":\"x86\"," 88fi 89 90cross_compile="" 91if [[ ${use_musl} = "true" && ${ARCH} = x86 ]]; then 92 cross_compile=$(cat <<EOF 93 "CrossHost": "linux_musl", 94 "CrossHostArch": "arm64", 95 "CrossHostSecondaryArch": "arm", 96EOF 97 ) 98elif [[ ${OS} = darwin ]]; then 99 cross_compile=$(cat <<EOF 100 "CrossHost": "darwin", 101 "CrossHostArch": "arm64", 102EOF 103 ) 104fi 105 106# Use toybox and other prebuilts even outside of the build (test running, go, etc) 107export PATH=${TOP}/prebuilts/build-tools/path/${OS}-${ARCH}:$PATH 108 109if [ -n "${build_soong}" ]; then 110 SOONG_OUT=${OUT_DIR}/soong 111 SOONG_HOST_OUT=${OUT_DIR}/host/${OS}-${ARCH} 112 [[ -z "${clean}" ]] || rm -rf ${SOONG_OUT} 113 mkdir -p ${SOONG_OUT} 114 rm -rf ${SOONG_OUT}/dist ${SOONG_OUT}/dist-common ${SOONG_OUT}/dist-arm64 115 cat > ${SOONG_OUT}/soong.variables.tmp << EOF 116{ 117 "Allow_missing_dependencies": true, 118 ${primary_arch} 119 ${secondary_arch} 120 ${cross_compile} 121 "HostMusl": $use_musl, 122 "StripByDefault": true, 123 "VendorVars": { 124 "cpython3": { 125 "force_build_host": "true" 126 }, 127 "art_module": { 128 "source_build": "true" 129 } 130 }, 131 "VendorVarTypes": { 132 "cpython3": { 133 "force_build_host": "bool" 134 } 135 } 136} 137EOF 138 if cmp -s ${SOONG_OUT}/soong.variables.tmp ${SOONG_OUT}/soong.variables; then 139 rm ${SOONG_OUT}/soong.variables.tmp 140 else 141 mv -f ${SOONG_OUT}/soong.variables.tmp ${SOONG_OUT}/soong.variables 142 fi 143 SOONG_GO_BINARIES=( 144 bpfmt 145 go_extractor 146 kotlinc_extractor 147 merge_zips 148 soong_zip 149 runextractor 150 rust_extractor 151 zip2zip 152 ) 153 SOONG_BINARIES=( 154 aconfig 155 acp 156 aidl 157 bison 158 bloaty 159 brotli 160 bzip2 161 ckati 162 edit_monitor 163 flex 164 gavinhoward-bc 165 hidl-gen 166 hidl-lint 167 m4 168 make 169 n2 170 ninja 171 one-true-awk 172 openssl 173 py3-cmd 174 py3-launcher64 175 py3-launcher-autorun64 176 tool_event_logger 177 toybox 178 xz 179 zipalign 180 ziptime 181 ziptool 182 ) 183 SOONG_MUSL_BINARIES=( 184 py3-launcher-static64 185 py3-launcher-autorun-static64 186 ) 187 SOONG_ASAN_BINARIES=( 188 acp 189 aidl 190 ckati 191 gavinhoward-bc 192 ninja 193 toybox 194 zipalign 195 ziptime 196 ziptool 197 ) 198 SOONG_JAVA_LIBRARIES=( 199 desugar.jar 200 dx.jar 201 javac_extractor.jar 202 ktfmt.jar 203 turbine.jar 204 ) 205 SOONG_JAVA_WRAPPERS=( 206 dx 207 ) 208 if [[ $OS == "linux" ]]; then 209 SOONG_BINARIES+=( 210 create_minidebuginfo 211 nsjail 212 ) 213 fi 214 215 go_binaries="${SOONG_GO_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 216 if [ -n "${build_native}" ]; then 217 binaries="${SOONG_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 218 asan_binaries="${SOONG_ASAN_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 219 tests="${SOONG_HOST_OUT}/nativetest64/n2_e2e_tests/n2_e2e_tests \ 220 ${SOONG_HOST_OUT}/nativetest64/n2_unit_tests/n2_unit_tests \ 221 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test \ 222 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test \ 223 ${SOONG_HOST_OUT}/nativetest64/par_test/par_test" 224 225 # TODO: When we have a better method of extracting zips from Soong, use that. 226 py3_stdlib_zip="${SOONG_OUT}/.intermediates/external/python/cpython3/Lib/py3-stdlib-zip/gen/py3-stdlib.zip" 227 fi 228 if [ -n "${build_java}" ]; then 229 jars="${SOONG_JAVA_LIBRARIES[@]/#/${SOONG_HOST_OUT}/framework/}" 230 wrappers="${SOONG_JAVA_WRAPPERS[@]/#/${SOONG_HOST_OUT}/bin/}" 231 fi 232 233 musl_x86_sysroot="" 234 musl_x86_64_sysroot="" 235 musl_arm_sysroot="" 236 musl_arm64_sysroot="" 237 cross_binaries="" 238 if [[ ${use_musl} = "true" && -n "${build_arm64_cross_musl}" ]]; then 239 binaries="${binaries} ${SOONG_MUSL_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 240 musl_x86_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_x86/gen/libc_musl_sysroot.zip" 241 musl_x86_64_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_x86_64/gen/libc_musl_sysroot.zip" 242 243 # Build cross-compiled musl arm64 binaries, except for go binaries 244 # since Blueprint doesn't have cross compilation support for them. 245 SOONG_HOST_ARM_OUT=${OUT_DIR}/host/linux-arm64 246 cross_binaries="${SOONG_BINARIES[@]/#/${SOONG_HOST_ARM_OUT}/bin/}" 247 cross_binaries="${cross_binaries} ${SOONG_MUSL_BINARIES[@]/#/${SOONG_HOST_ARM_OUT}/bin/}" 248 249 musl_arm_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_arm/gen/libc_musl_sysroot.zip" 250 musl_arm64_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_arm64/gen/libc_musl_sysroot.zip" 251 fi 252 253 # Build everything 254 build/soong/soong_ui.bash --make-mode --soong-only --skip-config ${skip_soong_tests} \ 255 ${go_binaries} \ 256 ${binaries} \ 257 ${cross_binaries} \ 258 ${wrappers} \ 259 ${jars} \ 260 ${py3_stdlib_zip} \ 261 ${musl_x86_sysroot} \ 262 ${musl_x86_64_sysroot} \ 263 ${musl_arm_sysroot} \ 264 ${musl_arm64_sysroot} \ 265 ${tests} \ 266 soong_docs 267 268 if [ -n "${build_native}" ]; then 269 # Run ninja tests 270 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test 271 272 # Run n2 tests 273 timeout -v -k 10 300 ${SOONG_HOST_OUT}/nativetest64/n2_unit_tests/n2_unit_tests 274 N2_PATH=${SOONG_HOST_OUT}/bin/n2 timeout -v -k 10 300 ${SOONG_HOST_OUT}/nativetest64/n2_e2e_tests/n2_e2e_tests 275 276 # Run ckati tests 277 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test 278 279 # Run python par/py*-cmd tests 280 ANDROID_HOST_OUT=${PWD}/${SOONG_HOST_OUT} build/soong/python/tests/runtest.sh 281 fi 282 283 # Copy arch-specific binaries 284 mkdir -p ${SOONG_OUT}/dist/bin 285 cp ${go_binaries} ${binaries} ${SOONG_OUT}/dist/bin/ 286 for i in $(ls -d ${SOONG_HOST_OUT}/lib* 2> /dev/null); do 287 cp -R ${i} ${SOONG_OUT}/dist/ 288 done 289 290 # Copy cross-compiled binaries 291 if [[ "${use_musl}" = "true" && -n "${build_arm64_cross_musl}" ]]; then 292 mkdir -p ${SOONG_OUT}/dist-arm64/bin 293 cp ${cross_binaries} ${SOONG_OUT}/dist-arm64/bin/ 294 cp -R ${SOONG_HOST_ARM_OUT}/lib* ${SOONG_OUT}/dist-arm64/ 295 fi 296 297 if [ -n "${build_java}" ]; then 298 # Copy jars and wrappers 299 mkdir -p ${SOONG_OUT}/dist-common/{bin,flex,framework,py3-stdlib,py3-headers} 300 cp ${wrappers} ${SOONG_OUT}/dist-common/bin 301 cp ${jars} ${SOONG_OUT}/dist-common/framework 302 fi 303 304 if [ -n "${build_native}" ]; then 305 cp -r external/bison/data ${SOONG_OUT}/dist-common/bison 306 cp external/bison/NOTICE ${SOONG_OUT}/dist-common/bison/ 307 cp -r external/flex/src/FlexLexer.h ${SOONG_OUT}/dist-common/flex/ 308 cp external/flex/NOTICE ${SOONG_OUT}/dist-common/flex/ 309 310 ${SOONG_OUT}/dist/bin/zip2zip -i ${py3_stdlib_zip} -o ${OUT_DIR}/py3-stdlib.zip "**/*:py3-stdlib/" 311 cp external/python/cpython3/LICENSE ${SOONG_OUT}/dist-common/py3-stdlib/ 312 py3_headers=( 313 ${TOP}/external/python/cpython3/Include 314 ${TOP}/external/python/cpython3/android/linux_x86_64/pyconfig 315 ${TOP}/external/python/cpython3/android/linux_arm64/pyconfig 316 ${TOP}/external/python/cpython3/android/bionic/pyconfig 317 ${TOP}/external/python/cpython3/android/darwin/pyconfig 318 ) 319 for py3_header in ${py3_headers[@]}; do 320 REL_SUBDIR=${py3_header#*/cpython3/} 321 REL_SUBDIR=$(dirname ${REL_SUBDIR}) 322 mkdir -p ${SOONG_OUT}/dist-common/py3-headers/${REL_SUBDIR} || true 323 cp -r ${py3_header} ${SOONG_OUT}/dist-common/py3-headers/${REL_SUBDIR}/ 324 done 325 cp external/python/cpython3/LICENSE ${SOONG_OUT}/dist-common/py3-headers/ 326 fi 327 328 if [[ "${use_musl}" = "true" && -n "${build_arm64_cross_musl}" ]]; then 329 cp ${musl_x86_64_sysroot} ${SOONG_OUT}/musl-sysroot-x86_64-unknown-linux-musl.zip 330 cp ${musl_x86_sysroot} ${SOONG_OUT}/musl-sysroot-i686-unknown-linux-musl.zip 331 cp ${musl_arm_sysroot} ${SOONG_OUT}/musl-sysroot-arm-unknown-linux-musleabihf.zip 332 cp ${musl_arm64_sysroot} ${SOONG_OUT}/musl-sysroot-aarch64-unknown-linux-musl.zip 333 fi 334 335 if [[ $OS == "linux" && -n "${build_asan}" ]]; then 336 # Build ASAN versions 337 export ASAN_OPTIONS=detect_leaks=0 338 cat > ${SOONG_OUT}/soong.variables << EOF 339{ 340 "Allow_missing_dependencies": true, 341 "HostArch":"x86_64", 342 ${secondary_arch} 343 "SanitizeHost": ["address"], 344 "VendorVars": { 345 "art_module": { 346 "source_build": "true" 347 } 348 } 349} 350EOF 351 352 export ASAN_SYMBOLIZER_PATH=${PWD}/prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer 353 354 # Clean up non-ASAN installed versions 355 rm -rf ${SOONG_HOST_OUT} 356 357 # Build everything with ASAN 358 build/soong/soong_ui.bash --make-mode --soong-only --skip-config ${skip_soong_tests} \ 359 ${asan_binaries} \ 360 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test \ 361 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test 362 363 # Run ninja tests 364 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test 365 366 # Run ckati tests 367 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test 368 369 # Copy arch-specific binaries 370 mkdir -p ${SOONG_OUT}/dist/asan/bin 371 cp ${asan_binaries} ${SOONG_OUT}/dist/asan/bin/ 372 cp -R ${SOONG_HOST_OUT}/lib* ${SOONG_OUT}/dist/asan/ 373 fi 374 375 # Package arch-specific prebuilts 376 ${SOONG_OUT}/dist/bin/soong_zip -o ${OUT_DIR}/build-prebuilts.zip -C ${SOONG_OUT}/dist -D ${SOONG_OUT}/dist 377 378 if [[ ${use_musl} = "true" && -n "${build_arm64_cross_musl}" ]]; then 379 # Package cross-compiled prebuilts 380 ${SOONG_OUT}/dist/bin/soong_zip -o ${OUT_DIR}/build-arm64-prebuilts.zip -C ${SOONG_OUT}/dist-arm64 -D ${SOONG_OUT}/dist-arm64 381 fi 382 383 if [[ -n "${build_java}" || -n "${build_native}" ]]; then 384 # Package common prebuilts 385 ${SOONG_OUT}/dist/bin/soong_zip -o ${OUT_DIR}/build-common-prebuilts.tmp.zip -C ${SOONG_OUT}/dist-common -D ${SOONG_OUT}/dist-common 386 ${SOONG_OUT}/dist/bin/merge_zips ${OUT_DIR}/build-common-prebuilts.zip ${OUT_DIR}/build-common-prebuilts.tmp.zip ${OUT_DIR}/py3-stdlib.zip 387 fi 388fi 389 390if [ -z "${skip_soong_tests}" ]; then 391 build/soong/scripts/run-soong-tests-with-go-tools.sh 392fi 393 394# Go 395if [ -n "${build_go}" ]; then 396 GO_OUT=${OUT_DIR}/obj/go 397 rm -rf ${GO_OUT} 398 mkdir -p ${GO_OUT} 399 cp -a ${TOP}/toolchain/go/* ${GO_OUT}/ 400 rm -f ${GO_OUT}/update_prebuilts.sh 401 ( 402 cd ${GO_OUT}/src 403 export GOROOT_BOOTSTRAP=${TOP}/prebuilts/go/${OS}-${ARCH} 404 export GOROOT_FINAL=./prebuilts/go/${OS}-${ARCH} 405 export GO_TEST_TIMEOUT_SCALE=100 406 export GODEBUG=installgoroot=all 407 ./make.bash 408 rm -rf ../pkg/bootstrap 409 rm -rf ../pkg/obj 410 GOROOT=$(pwd)/.. ../bin/go install -race std 411 ) 412 ${SOONG_OUT}/dist/bin/soong_zip -o ${OUT_DIR}/go.zip -C ${GO_OUT} -D ${GO_OUT} 413fi 414 415if [ -n "${DIST_DIR}" ]; then 416 mkdir -p ${DIST_DIR} || true 417 418 if [ -n "${build_soong}" ]; then 419 cp ${OUT_DIR}/build-prebuilts.zip ${DIST_DIR}/ 420 if [ -n "${build_java}" ]; then 421 cp ${OUT_DIR}/build-common-prebuilts.zip ${DIST_DIR}/ 422 fi 423 cp ${SOONG_OUT}/docs/*.html ${DIST_DIR}/ 424 if [[ ${use_musl} = "true" && -n "${build_arm64_cross_musl}" ]]; then 425 cp ${OUT_DIR}/build-arm64-prebuilts.zip ${DIST_DIR}/ 426 427 cp ${SOONG_OUT}/musl-sysroot-x86_64-unknown-linux-musl.zip ${DIST_DIR}/ 428 cp ${SOONG_OUT}/musl-sysroot-i686-unknown-linux-musl.zip ${DIST_DIR}/ 429 cp ${SOONG_OUT}/musl-sysroot-arm-unknown-linux-musleabihf.zip ${DIST_DIR}/ 430 cp ${SOONG_OUT}/musl-sysroot-aarch64-unknown-linux-musl.zip ${DIST_DIR}/ 431 fi 432 fi 433 if [ -n "${build_go}" ]; then 434 cp ${OUT_DIR}/go.zip ${DIST_DIR}/ 435 fi 436fi 437