1#!/bin/bash -eux 2# Copyright 2016 Google Inc. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17 18# See issue #4270. The compiler crashes on GCB instance with 32 vCPUs, so when 19# we compile on GCB we want 16 cores. But locally we want more (so use nproc / 20# 2). 21NPROC=$(expr $(nproc) / 2) 22 23# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler 24LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 python3-distutils g++-multilib binutils-dev zlib1g-dev" 25apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends 26 27# Checkout 28CHECKOUT_RETRIES=10 29function clone_with_retries { 30 REPOSITORY=$1 31 LOCAL_PATH=$2 32 CHECKOUT_RETURN_CODE=1 33 34 # Disable exit on error since we might encounter some failures while retrying. 35 set +e 36 for i in $(seq 1 $CHECKOUT_RETRIES); do 37 rm -rf $LOCAL_PATH 38 git clone $REPOSITORY $LOCAL_PATH 39 CHECKOUT_RETURN_CODE=$? 40 if [ $CHECKOUT_RETURN_CODE -eq 0 ]; then 41 break 42 fi 43 done 44 45 # Re-enable exit on error. If checkout failed, script will exit. 46 set -e 47 return $CHECKOUT_RETURN_CODE 48} 49 50function cmake_llvm { 51 extra_args="$@" 52 cmake -G "Ninja" \ 53 -DLIBCXX_ENABLE_SHARED=OFF \ 54 -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ 55 -DLIBCXXABI_ENABLE_SHARED=OFF \ 56 -DCMAKE_BUILD_TYPE=Release \ 57 -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \ 58 -DLLVM_ENABLE_PROJECTS="$PROJECTS_TO_BUILD" \ 59 -DLLVM_BINUTILS_INCDIR="/usr/include/" \ 60 $extra_args \ 61 $LLVM_SRC/llvm 62} 63 64# Use chromium's clang revision 65mkdir $SRC/chromium_tools 66cd $SRC/chromium_tools 67git clone https://chromium.googlesource.com/chromium/src/tools/clang --depth 1 68cd clang 69 70LLVM_SRC=$SRC/llvm-project 71 72# For manual bumping. 73OUR_LLVM_REVISION=llvmorg-12-init-17251-g6de48655 74 75# To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e. 76# *not* force a manual downgrade). Set to 1 to force a manual downgrade. 77FORCE_OUR_REVISION=0 78LLVM_REVISION=$(grep -Po "CLANG_REVISION = '\K([^']+)" scripts/update.py) 79 80clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC 81 82set +e 83git -C $LLVM_SRC merge-base --is-ancestor $OUR_LLVM_REVISION $LLVM_REVISION 84IS_OUR_REVISION_ANCESTOR_RETCODE=$? 85set -e 86 87# Use our revision if specified by FORCE_OUR_REVISION or if our revision is a 88# later revision than Chrome's (i.e. not an ancestor of Chrome's). 89if [ $IS_OUR_REVISION_ANCESTOR_RETCODE -ne 0 ] || [ $FORCE_OUR_REVISION -eq 1 ] ; then 90 LLVM_REVISION=$OUR_LLVM_REVISION 91fi 92 93git -C $LLVM_SRC checkout $LLVM_REVISION 94echo "Using LLVM revision: $LLVM_REVISION" 95 96# Build & install. 97mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1 98python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1 99 100TARGET_TO_BUILD= 101case $(uname -m) in 102 x86_64) 103 TARGET_TO_BUILD=X86 104 ;; 105 aarch64) 106 TARGET_TO_BUILD=AArch64 107 ;; 108 *) 109 echo "Error: unsupported target $(uname -m)" 110 exit 1 111 ;; 112esac 113 114PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld" 115 116cd $WORK/llvm-stage2 117export CC=$WORK/llvm-stage1/bin/clang 118export CXX=$WORK/llvm-stage1/bin/clang++ 119cmake_llvm 120ninja -j $NPROC 121ninja install 122rm -rf $WORK/llvm-stage1 $WORK/llvm-stage2 123 124# Use the clang we just built from now on. 125CMAKE_EXTRA_ARGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" 126 127function cmake_libcxx { 128 extra_args="$@" 129 cmake -G "Ninja" \ 130 -DLIBCXX_ENABLE_SHARED=OFF \ 131 -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ 132 -DLIBCXXABI_ENABLE_SHARED=OFF \ 133 -DCMAKE_BUILD_TYPE=Release \ 134 -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \ 135 -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \ 136 -DLLVM_BINUTILS_INCDIR="/usr/include/" \ 137 $extra_args \ 138 $LLVM_SRC/llvm 139} 140 141# 32-bit libraries. 142mkdir -p $WORK/i386 143cd $WORK/i386 144cmake_libcxx $CMAKE_EXTRA_ARGS \ 145 -DCMAKE_INSTALL_PREFIX=/usr/i386/ \ 146 -DCMAKE_C_FLAGS="-m32" \ 147 -DCMAKE_CXX_FLAGS="-m32" 148 149ninja -j $NPROC cxx 150ninja install-cxx 151rm -rf $WORK/i386 152 153# MemorySanitizer instrumented libraries. 154mkdir -p $WORK/msan 155cd $WORK/msan 156 157# https://github.com/google/oss-fuzz/issues/1099 158cat <<EOF > $WORK/msan/blocklist.txt 159fun:__gxx_personality_* 160EOF 161 162cmake_libcxx $CMAKE_EXTRA_ARGS \ 163 -DLLVM_USE_SANITIZER=Memory \ 164 -DCMAKE_INSTALL_PREFIX=/usr/msan/ \ 165 -DCMAKE_CXX_FLAGS="-fsanitize-blacklist=$WORK/msan/blocklist.txt" 166 167ninja -j $NPROC cxx 168ninja install-cxx 169rm -rf $WORK/msan 170 171# DataFlowSanitizer instrumented libraries. 172mkdir -p $WORK/dfsan 173cd $WORK/dfsan 174 175cmake_libcxx $CMAKE_EXTRA_ARGS \ 176 -DLLVM_USE_SANITIZER=DataFlow \ 177 -DCMAKE_INSTALL_PREFIX=/usr/dfsan/ 178 179ninja -j $NPROC cxx cxxabi 180ninja install-cxx install-cxxabi 181rm -rf $WORK/dfsan 182 183# libFuzzer sources. 184cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer 185 186# Cleanup 187rm -rf $LLVM_SRC 188rm -rf $SRC/chromium_tools 189apt-get remove --purge -y $LLVM_DEP_PACKAGES 190apt-get autoremove -y 191 192# Delete unneeded parts of LLVM to reduce image size. 193# See https://github.com/google/oss-fuzz/issues/5170 194LLVM_TOOLS_TMPDIR=/tmp/llvm-tools 195mkdir $LLVM_TOOLS_TMPDIR 196# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR 197mv \ 198 /usr/local/bin/llvm-ar \ 199 /usr/local/bin/llvm-as \ 200 /usr/local/bin/llvm-config \ 201 /usr/local/bin/llvm-cov \ 202 /usr/local/bin/llvm-objcopy \ 203 /usr/local/bin/llvm-profdata \ 204 /usr/local/bin/llvm-ranlib \ 205 /usr/local/bin/llvm-symbolizer \ 206 /usr/local/bin/llvm-undname \ 207 $LLVM_TOOLS_TMPDIR 208# Delete remaining llvm- binaries. 209rm -rf /usr/local/bin/llvm-* 210# Restore the llvm- binaries we want to keep. 211mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/ 212rm -rf $LLVM_TOOLS_TMPDIR 213 214# Remove binaries from LLVM build that we don't need. 215rm -f \ 216 /usr/local/bin/bugpoint \ 217 /usr/local/bin/llc \ 218 /usr/local/bin/lli \ 219 /usr/local/bin/clang-check \ 220 /usr/local/bin/clang-refactor \ 221 /usr/local/bin/clang-offload-wrapper \ 222 /usr/local/bin/clang-offload-bundler \ 223 /usr/local/bin/clang-check \ 224 /usr/local/bin/clang-refactor \ 225 /usr/local/bin/c-index-test \ 226 /usr/local/bin/clang-rename \ 227 /usr/local/bin/clang-scan-deps \ 228 /usr/local/bin/clang-extdef-mapping \ 229 /usr/local/bin/diagtool \ 230 /usr/local/bin/sanstats \ 231 /usr/local/bin/dsymutil \ 232 /usr/local/bin/verify-uselistorder \ 233 /usr/local/bin/clang-format 234 235# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the 236# libraries. 237# Note: we need fuzzer_no_main libraries for atheris. Don't delete. 238rm -rf \ 239 /usr/local/lib/libclang* \ 240 /usr/local/lib/liblld* \ 241 /usr/local/lib/cmake/ 242