1#!/bin/bash -eux 2# Copyright 2016 Google Inc. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17 18# See issue #4270. The compiler crashes on GCB instance with 32 vCPUs, so when 19# we compile on GCB we want 16 cores. But locally we want more (so use nproc / 20# 2). 21NPROC=$(expr $(nproc) / 2) 22 23# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler 24LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev" 25apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends 26 27# Checkout 28CHECKOUT_RETRIES=10 29function clone_with_retries { 30 REPOSITORY=$1 31 LOCAL_PATH=$2 32 CHECKOUT_RETURN_CODE=1 33 34 # Disable exit on error since we might encounter some failures while retrying. 35 set +e 36 for i in $(seq 1 $CHECKOUT_RETRIES); do 37 rm -rf $LOCAL_PATH 38 git clone $REPOSITORY $LOCAL_PATH 39 CHECKOUT_RETURN_CODE=$? 40 if [ $CHECKOUT_RETURN_CODE -eq 0 ]; then 41 break 42 fi 43 done 44 45 # Re-enable exit on error. If checkout failed, script will exit. 46 set -e 47 return $CHECKOUT_RETURN_CODE 48} 49 50function cmake_llvm { 51 extra_args="$@" 52 cmake -G "Ninja" \ 53 -DLIBCXX_ENABLE_SHARED=OFF \ 54 -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ 55 -DLIBCXXABI_ENABLE_SHARED=OFF \ 56 -DCMAKE_BUILD_TYPE=Release \ 57 -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \ 58 -DLLVM_ENABLE_PROJECTS="$PROJECTS_TO_BUILD" \ 59 -DLLVM_BINUTILS_INCDIR="/usr/include/" \ 60 $extra_args \ 61 $LLVM_SRC/llvm 62} 63 64# Use chromium's clang revision 65mkdir $SRC/chromium_tools 66cd $SRC/chromium_tools 67git clone https://chromium.googlesource.com/chromium/src/tools/clang --depth 1 68cd clang 69 70LLVM_SRC=$SRC/llvm-project 71 72# For manual bumping. 73OUR_LLVM_REVISION=llvmorg-12-init-17251-g6de48655 74 75# To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e. 76# *not* force a manual downgrade). Set to 1 to force a manual downgrade. 77FORCE_OUR_REVISION=1 78LLVM_REVISION=$(grep -Po "CLANG_REVISION = '\K([^']+)" scripts/update.py) 79 80clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC 81 82set +e 83git -C $LLVM_SRC merge-base --is-ancestor $OUR_LLVM_REVISION $LLVM_REVISION 84IS_OUR_REVISION_ANCESTOR_RETCODE=$? 85set -e 86 87# Use our revision if specified by FORCE_OUR_REVISION or if our revision is a 88# later revision than Chrome's (i.e. not an ancestor of Chrome's). 89if [ $IS_OUR_REVISION_ANCESTOR_RETCODE -ne 0 ] || [ $FORCE_OUR_REVISION -eq 1 ] ; then 90 LLVM_REVISION=$OUR_LLVM_REVISION 91fi 92 93git -C $LLVM_SRC checkout $LLVM_REVISION 94echo "Using LLVM revision: $LLVM_REVISION" 95 96# Build & install. 97mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1 98python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1 99 100TARGET_TO_BUILD= 101case $(uname -m) in 102 x86_64) 103 TARGET_TO_BUILD=X86 104 ;; 105 aarch64) 106 TARGET_TO_BUILD=AArch64 107 ;; 108 *) 109 echo "Error: unsupported target $(uname -m)" 110 exit 1 111 ;; 112esac 113 114PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld" 115 116cd $WORK/llvm-stage2 117export CC=$WORK/llvm-stage1/bin/clang 118export CXX=$WORK/llvm-stage1/bin/clang++ 119cmake_llvm 120ninja -j $NPROC 121ninja install 122rm -rf $WORK/llvm-stage1 $WORK/llvm-stage2 123 124# Use the clang we just built from now on. 125CMAKE_EXTRA_ARGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" 126 127# 32-bit libraries. 128mkdir -p $WORK/i386 129cd $WORK/i386 130cmake_llvm $CMAKE_EXTRA_ARGS \ 131 -DCMAKE_INSTALL_PREFIX=/usr/i386/ \ 132 -DCMAKE_C_FLAGS="-m32" \ 133 -DCMAKE_CXX_FLAGS="-m32" 134 135ninja -j $NPROC cxx 136ninja install-cxx 137rm -rf $WORK/i386 138 139# MemorySanitizer instrumented libraries. 140mkdir -p $WORK/msan 141cd $WORK/msan 142 143# https://github.com/google/oss-fuzz/issues/1099 144cat <<EOF > $WORK/msan/blocklist.txt 145fun:__gxx_personality_* 146EOF 147 148cmake_llvm $CMAKE_EXTRA_ARGS \ 149 -DLLVM_USE_SANITIZER=Memory \ 150 -DCMAKE_INSTALL_PREFIX=/usr/msan/ \ 151 -DCMAKE_CXX_FLAGS="-fsanitize-blacklist=$WORK/msan/blocklist.txt" 152 153ninja -j $NPROC cxx 154ninja install-cxx 155rm -rf $WORK/msan 156 157# DataFlowSanitizer instrumented libraries. 158mkdir -p $WORK/dfsan 159cd $WORK/dfsan 160 161cmake_llvm $CMAKE_EXTRA_ARGS \ 162 -DLLVM_USE_SANITIZER=DataFlow \ 163 -DCMAKE_INSTALL_PREFIX=/usr/dfsan/ 164 165ninja -j $NPROC cxx cxxabi 166ninja install-cxx install-cxxabi 167rm -rf $WORK/dfsan 168 169# libFuzzer sources. 170cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer 171 172# Cleanup 173rm -rf $LLVM_SRC 174rm -rf $SRC/chromium_tools 175apt-get remove --purge -y $LLVM_DEP_PACKAGES 176apt-get autoremove -y 177 178# Delete unneeded parts of LLVM to reduce image size. 179# See https://github.com/google/oss-fuzz/issues/5170 180LLVM_TOOLS_TMPDIR=/tmp/llvm-tools 181mkdir $LLVM_TOOLS_TMPDIR 182# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR 183mv \ 184 /usr/local/bin/llvm-ar \ 185 /usr/local/bin/llvm-as \ 186 /usr/local/bin/llvm-config \ 187 /usr/local/bin/llvm-cov \ 188 /usr/local/bin/llvm-objcopy \ 189 /usr/local/bin/llvm-profdata \ 190 /usr/local/bin/llvm-ranlib \ 191 /usr/local/bin/llvm-symbolizer \ 192 /usr/local/bin/llvm-undname \ 193 $LLVM_TOOLS_TMPDIR 194# Delete remaining llvm- binaries. 195rm -rf /usr/local/bin/llvm-* 196# Restore the llvm- binaries we want to keep. 197mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/ 198rm -rf $LLVM_TOOLS_TMPDIR 199 200# Remove binaries from LLVM buld that we don't need. 201rm -f \ 202 /usr/local/bin/bugpoint \ 203 /usr/local/bin/llc \ 204 /usr/local/bin/lli \ 205 /usr/local/bin/clang-check \ 206 /usr/local/bin/clang-refactor \ 207 /usr/local/bin/clang-offload-wrapper \ 208 /usr/local/bin/clang-offload-bundler \ 209 /usr/local/bin/clang-check \ 210 /usr/local/bin/clang-refactor \ 211 /usr/local/bin/c-index-test \ 212 /usr/local/bin/clang-rename \ 213 /usr/local/bin/clang-scan-deps \ 214 /usr/local/bin/clang-extdef-mapping \ 215 /usr/local/bin/diagtool \ 216 /usr/local/bin/sanstats \ 217 /usr/local/bin/dsymutil \ 218 /usr/local/bin/verify-uselistorder \ 219 /usr/local/bin/clang-format 220 221# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the 222# libraries. 223# Note: we need fuzzer_no_main libraries for atheris. Don't delete. 224rm -rf \ 225 /usr/local/lib/libclang* \ 226 /usr/local/lib/liblld* \ 227 /usr/local/lib/cmake/ 228