1#!/bin/bash 2 3# Note: not intended to be invoked directly, see rebuild.sh. 4# 5# Rebuilds Crosvm and its dependencies from a clean state. 6 7: ${TOOLS_DIR:="$(pwd)/tools"} 8 9# Stable is usually too old for crosvm, but make sure you bump this 10# up as far as you can each time this script is touched.. 11RUST_TOOLCHAIN_VER=1.65.0 12 13setup_env() { 14 : ${SOURCE_DIR:="$(pwd)/source"} 15 : ${WORKING_DIR:="$(pwd)/working"} 16 : ${CUSTOM_MANIFEST:=""} 17 18 ARCH="$(uname -m)" 19 : ${OUTPUT_DIR:="$(pwd)/${ARCH}-linux-gnu"} 20 OUTPUT_BIN_DIR="${OUTPUT_DIR}/bin" 21 OUTPUT_ETC_DIR="${OUTPUT_DIR}/etc" 22 OUTPUT_SECCOMP_DIR="${OUTPUT_ETC_DIR}/seccomp" 23 24 export PATH="${PATH}:${TOOLS_DIR}:${HOME}/.local/bin" 25 export PKG_CONFIG_PATH="${WORKING_DIR}/usr/lib/pkgconfig" 26} 27 28set -e 29set -x 30 31fatal_echo() { 32 echo "$@" 33 exit 1 34} 35 36prepare_cargo() { 37 echo Setting up cargo... 38 cd 39 rm -rf .cargo 40 # Sometimes curl hangs. When it does, retry 41 retry curl -L \ 42 --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init 43 chmod +x rustup-init 44 ./rustup-init -y --no-modify-path --default-toolchain ${RUST_TOOLCHAIN_VER} 45 source $HOME/.cargo/env 46 if [[ -n "$1" ]]; then 47 rustup target add "$1" 48 fi 49 rm -f rustup-init 50 51 if [[ -n "$1" ]]; then 52 cat >>~/.cargo/config <<EOF 53[target.$1] 54linker = "${1/-unknown-/-}" 55EOF 56 fi 57} 58 59install_packages() { 60 echo Installing packages... 61 62 sudo dpkg --add-architecture arm64 63 sudo apt-get update 64 sudo apt-get install -y \ 65 "$@" \ 66 autoconf \ 67 automake \ 68 build-essential \ 69 clang \ 70 curl \ 71 doxygen \ 72 g++ \ 73 gcc \ 74 git \ 75 graphviz \ 76 libcap-dev \ 77 libegl1-mesa-dev \ 78 libfdt-dev \ 79 libgl1-mesa-dev \ 80 libgles2-mesa-dev \ 81 libpciaccess-dev \ 82 libssl-dev \ 83 libtool \ 84 libusb-1.0-0-dev \ 85 libwayland-bin \ 86 libwayland-dev \ 87 libxml2-dev \ 88 make \ 89 nasm \ 90 ninja-build \ 91 pkg-config \ 92 protobuf-compiler \ 93 python3 \ 94 python3-pip \ 95 texinfo \ 96 wayland-protocols \ 97 xmlto \ 98 xutils-dev # Needed to pacify autogen.sh for libepoxy 99 mkdir -p "${TOOLS_DIR}" 100 101 # Repo needs python3 but python-is-python3 package not available: 102 sudo ln -s -f /usr/bin/python3 /usr/bin/python 103 104 curl https://storage.googleapis.com/git-repo-downloads/repo > "${TOOLS_DIR}/repo" 105 chmod a+x "${TOOLS_DIR}/repo" 106 107 # Gfxstream needs a new-ish version of CMake 108 mkdir -p "${TOOLS_DIR}/cmake" 109 cd "${TOOLS_DIR}/cmake" 110 curl -O -L https://cmake.org/files/v3.22/cmake-3.22.1-linux-$(uname -m).sh 111 chmod +x cmake-3.22.1-linux-$(uname -m).sh 112 sudo ./cmake-3.22.1-linux-$(uname -m).sh --skip-license --exclude-subdir --prefix=/usr/local 113 cmake --version 114 115 # Meson getting started guide mentions that the distro version is frequently 116 # outdated and recommends installing via pip. 117 pip3 install --no-warn-script-location meson 118 119 # Tools for building gfxstream 120 pip3 install absl-py 121 pip3 install urlfetch 122 123 case "$(uname -m)" in 124 aarch64) 125 prepare_cargo 126 ;; 127 x86_64) 128 # Cross-compilation is x86_64 specific 129 sudo apt install -y crossbuild-essential-arm64 130 prepare_cargo aarch64-unknown-linux-gnu 131 ;; 132 esac 133} 134 135retry() { 136 for i in $(seq 5); do 137 "$@" && return 0 138 sleep 1 139 done 140 return 1 141} 142 143fetch_source() { 144 echo "Fetching source..." 145 146 mkdir -p "${SOURCE_DIR}" 147 cd "${SOURCE_DIR}" 148 149 if ! git config user.name; then 150 git config --global user.name "AOSP Crosvm Builder" 151 git config --global user.email "nobody@android.com" 152 git config --global color.ui false 153 fi 154 155 if [[ -z "${CUSTOM_MANIFEST}" ]]; then 156 # Building Crosvm currently depends using Chromium's directory scheme for subproject 157 # directories ('third_party' vs 'external'). 158 fatal_echo "CUSTOM_MANIFEST must be provided. You most likely want to provide a full path to" \ 159 "a copy of device/google/cuttlefish_vmm/manifest.xml." 160 fi 161 162 cp ${CUSTOM_MANIFEST} manifest.xml 163 repo init --depth=1 -q -u https://android.googlesource.com/platform/manifest -m $PWD/manifest.xml 164 repo sync 165} 166 167prepare_source() { 168 if [ "$(ls -A $SOURCE_DIR)" ]; then 169 echo "${SOURCE_DIR} is non empty. Run this from an empty directory if you wish to fetch the source." 1>&2 170 exit 2 171 fi 172 fetch_source 173} 174 175resync_source() { 176 echo "Deleting source directory..." 177 rm -rf "${SOURCE_DIR}/.*" 178 rm -rf "${SOURCE_DIR}/*" 179 fetch_source 180} 181 182# $1 = installed library filename 183debuglink() { 184 objcopy --only-keep-debug "${OUTPUT_BIN_DIR}/$1" "${OUTPUT_BIN_DIR}/$1.debug" 185 strip --strip-debug "${OUTPUT_BIN_DIR}/$1" 186 cd "${OUTPUT_BIN_DIR}" 187 objcopy --add-gnu-debuglink="$1.debug" "$1" 188 cd - 189} 190 191compile_libdrm() { 192 cd "${SOURCE_DIR}/external/libdrm" 193 194 # Ensure pkg-config file supplies rpath to dependent libraries 195 grep "install_rpath" meson.build || \ 196 sed -i "s|install : true,|install : true, install_rpath : '\$ORIGIN',|" meson.build 197 198 meson build \ 199 --libdir="${WORKING_DIR}/usr/lib" \ 200 --prefix="${WORKING_DIR}/usr" \ 201 -Damdgpu=false \ 202 -Dfreedreno=false \ 203 -Dintel=false \ 204 -Dlibkms=false \ 205 -Dnouveau=false \ 206 -Dradeon=false \ 207 -Dvc4=false \ 208 -Dvmwgfx=false 209 210 cd build 211 ninja install 212 213 cp -a "${WORKING_DIR}"/usr/lib/libdrm.so* "${OUTPUT_BIN_DIR}" 214 debuglink libdrm.so.2.4.0 215} 216 217compile_minijail() { 218 echo "Compiling Minijail..." 219 220 cd "${SOURCE_DIR}/platform/minijail" 221 222 # Disable warnings that the old gcc does not know about... 223 sed -i "s|-Wstring-compare||" common.mk 224 225 if ! grep '^# cuttlefish_vmm-rebuild-mark' Makefile; then 226 # Link minijail-sys rust crate dynamically to minijail 227 sed -i '/BUILD_STATIC_LIBS/d' rust/minijail-sys/build.rs 228 sed -i 's,static=minijail.pic,dylib=minijail,' rust/minijail-sys/build.rs 229 230 # Use Android prebuilt C files instead of generating them 231 sed -i 's,\(.*\.gen\.c: \),DISABLED_\1,' Makefile 232 cat >>Makefile <<EOF 233libconstants.gen.c: \$(SRC)/linux-x86/libconstants.gen.c 234 @cp \$< \$@ 235libsyscalls.gen.c: \$(SRC)/linux-x86/libsyscalls.gen.c 236 @cp \$< \$@ 237# cuttlefish_vmm-rebuild-mark 238EOF 239 fi 240 241 make -j OUT="${WORKING_DIR}" 242 cp "${WORKING_DIR}"/libminijail.so "${WORKING_DIR}"/usr/lib 243 244 cp -a "${WORKING_DIR}"/usr/lib/libminijail.so "${OUTPUT_BIN_DIR}" 245 debuglink libminijail.so 246} 247 248compile_minigbm() { 249 echo "Compiling Minigbm..." 250 251 cd "${SOURCE_DIR}/platform/minigbm" 252 253 # Minigbm's package config file has a default hard-coded path. Update here so 254 # that dependent packages can find the files. 255 sed -i "s|prefix=/usr\$|prefix=${WORKING_DIR}/usr|" gbm.pc 256 257 # The gbm used by upstream linux distros is not compatible with crosvm, which must use Chrome OS's 258 # minigbm. 259 local cpp_flags=(-I/working/usr/include -I/working/usr/include/libdrm) 260 local ld_flags=(-Wl,-soname,libgbm.so.1 -Wl,-rpath,\\\$\$ORIGIN -L/working/usr/lib) 261 local make_flags=() 262 local minigbm_drv=(${MINIGBM_DRV}) 263 for drv in "${minigbm_drv[@]}"; do 264 cpp_flags+=(-D"DRV_${drv}") 265 make_flags+=("DRV_${drv}"=1) 266 done 267 268 make -j install \ 269 "${make_flags[@]}" \ 270 CPPFLAGS="${cpp_flags[*]}" \ 271 DESTDIR="${WORKING_DIR}" \ 272 LDFLAGS="${ld_flags[*]}" \ 273 OUT="${WORKING_DIR}" 274 275 cp -a "${WORKING_DIR}"/usr/lib/libminigbm.so* "${OUTPUT_BIN_DIR}" 276 cp -a "${WORKING_DIR}"/usr/lib/libgbm.so* "${OUTPUT_BIN_DIR}" 277 debuglink libminigbm.so.1.0.0 278} 279 280compile_epoxy() { 281 cd "${SOURCE_DIR}/third_party/libepoxy" 282 283 meson build \ 284 --libdir="${WORKING_DIR}/usr/lib" \ 285 --prefix="${WORKING_DIR}/usr" \ 286 -Dglx=no \ 287 -Dx11=false \ 288 -Degl=yes 289 290 cd build 291 ninja install 292 293 cp -a "${WORKING_DIR}"/usr/lib/libepoxy.so* "${OUTPUT_BIN_DIR}" 294 debuglink libepoxy.so.0.0.0 295} 296 297compile_virglrenderer() { 298 echo "Compiling VirglRenderer..." 299 300 # Note: depends on libepoxy 301 cd "${SOURCE_DIR}/third_party/virglrenderer" 302 303 # Meson needs to have dependency information for header lookup. 304 sed -i "s|cc.has_header('epoxy/egl.h')|cc.has_header('epoxy/egl.h', dependencies: epoxy_dep)|" meson.build 305 306 # Ensure pkg-config file supplies rpath to dependent libraries 307 grep "install_rpath" src/meson.build || \ 308 sed -i "s|install : true|install : true, install_rpath : '\$ORIGIN'|" src/meson.build 309 310 meson build \ 311 --libdir="${WORKING_DIR}/usr/lib" \ 312 --prefix="${WORKING_DIR}/usr" \ 313 -Dplatforms=egl \ 314 -Dminigbm_allocation=false 315 316 cd build 317 ninja install 318 319 cp -a "${WORKING_DIR}"/usr/lib/libvirglrenderer.so* "${OUTPUT_BIN_DIR}" 320 debuglink libvirglrenderer.so.1.7.7 321} 322 323compile_libffi() { 324 cd "${SOURCE_DIR}/third_party/libffi" 325 326 ./autogen.sh 327 ./configure \ 328 --prefix="${WORKING_DIR}/usr" \ 329 --libdir="${WORKING_DIR}/usr/lib" 330 make && make check && make install 331 332 cp -a "${WORKING_DIR}"/usr/lib/libffi.so* "${OUTPUT_BIN_DIR}" 333 debuglink libffi.so.7.1.0 334} 335 336compile_wayland() { 337 cd "${SOURCE_DIR}/third_party/wayland" 338 339 # Need to figure out the right way to pass this down... 340 sed -i "s|install: true\$|install: true, install_rpath : '\$ORIGIN'|" src/meson.build 341 342 meson build \ 343 --libdir="${WORKING_DIR}/usr/lib" \ 344 --prefix="${WORKING_DIR}/usr" 345 ninja -C build/ install 346 347 cp -a "${WORKING_DIR}"/usr/lib/libwayland-client.so* "${OUTPUT_BIN_DIR}" 348 debuglink libwayland-client.so.0.3.0 349} 350 351compile_gfxstream() { 352 echo "Compiling gfxstream..." 353 354 local dist_dir="${SOURCE_DIR}/hardware/google/gfxstream/build" 355 [ -d "${dist_dir}" ] && rm -rf "${dist_dir}" 356 mkdir "${dist_dir}" 357 cd "${dist_dir}" 358 359 cmake .. \ 360 -G Ninja \ 361 -DBUILD_GRAPHICS_DETECTOR=ON \ 362 -DDEPENDENCY_RESOLUTION=AOSP \ 363 -DGFXSTREAM_ENABLE_HOST_TRACING=ON 364 365 ninja 366 367 chmod +x "${dist_dir}"/gfxstream_graphics_detector 368 cp -a "${dist_dir}"/gfxstream_graphics_detector "${OUTPUT_BIN_DIR}" 369 debuglink gfxstream_graphics_detector 370 371 chmod +x "${dist_dir}"/libgfxstream_backend.so 372 cp -a "${dist_dir}"/libgfxstream_backend.so "${WORKING_DIR}"/usr/lib 373 cp -a "${WORKING_DIR}"/usr/lib/libgfxstream_backend.so "${OUTPUT_BIN_DIR}" 374 debuglink libgfxstream_backend.so 375} 376 377compile_swiftshader() { 378 echo "Compiling SwiftShader..." 379 380 local dist_dir="${SOURCE_DIR}/external/swiftshader/build" 381 [ -d "${dist_dir}" ] && rm -rf "${dist_dir}" 382 mkdir -p "${dist_dir}" 383 cd "${dist_dir}" 384 385 CC=/usr/bin/clang \ 386 CXX=/usr/bin/clang++ \ 387 cmake .. \ 388 -G Ninja \ 389 -DSWIFTSHADER_BUILD_TESTS=FALSE \ 390 -DSWIFTSHADER_LLVM_VERSION=16.0 391 392 ninja 393 394 cp -a "${dist_dir}"/libvk_swiftshader.so "${OUTPUT_BIN_DIR}" 395 debuglink libvk_swiftshader.so 396 397 cp -a "${dist_dir}"/Linux/vk_swiftshader_icd.json "${OUTPUT_BIN_DIR}" 398} 399 400compile_crosvm() { 401 echo "Compiling Crosvm..." 402 403 source "${HOME}/.cargo/env" 404 405 # Workaround for aosp/1412815 406 cd "${SOURCE_DIR}/platform/crosvm/protos/src" 407 if ! grep '^mod generated {$' lib.rs; then 408 cat >>lib.rs <<EOF 409mod generated { 410 include!(concat!(env!("OUT_DIR"), "/generated.rs")); 411} 412EOF 413 fi 414 sed -i "s/pub use cdisk_spec_proto::cdisk_spec/pub use generated::cdisk_spec/" lib.rs 415 416 cd "${SOURCE_DIR}/platform/crosvm" 417 418 # Workaround for minijail-sys prepending -L/usr/lib/$arch dir 419 # which breaks the preferred search path for libdrm.so 420 sed -i '0,/pkg_config::Config::new().probe("libdrm")?;/{/pkg_config::Config::new().probe("libdrm")?;/d;}' rutabaga_gfx/build.rs 421 422 # Workaround rutabaga build thinking it needs at later version of virglrenderer. 423 sed -i 's/atleast_version("1.0.0")/atleast_version("0.10.0")/g' rutabaga_gfx/build.rs 424 425 local crosvm_features=audio,gdb,gpu,composite-disk,usb,virgl_renderer 426 if [[ $BUILD_GFXSTREAM -eq 1 ]]; then 427 crosvm_features+=,gfxstream 428 fi 429 430 CROSVM_USE_SYSTEM_MINIGBM=1 \ 431 CROSVM_USE_SYSTEM_VIRGLRENDERER=1 \ 432 GFXSTREAM_PATH="${WORKING_DIR}/usr/lib" \ 433 PKG_CONFIG_PATH="${WORKING_DIR}/usr/lib/pkgconfig" \ 434 RUSTFLAGS="-C link-arg=-Wl,-rpath,\$ORIGIN -C link-arg=${WORKING_DIR}/usr/lib/libdrm.so" \ 435 cargo build --features ${crosvm_features} 436 437 # Save the outputs 438 cp Cargo.lock "${OUTPUT_DIR}" 439 cp target/debug/crosvm "${OUTPUT_BIN_DIR}" 440 debuglink crosvm 441 442 cargo --version --verbose > "${OUTPUT_DIR}/cargo_version.txt" 443 rustup show > "${OUTPUT_DIR}/rustup_show.txt" 444} 445 446compile_crosvm_seccomp() { 447 echo "Processing Crosvm Seccomp..." 448 449 cd "${SOURCE_DIR}/platform/crosvm" 450 case ${ARCH} in 451 x86_64) subdir="${ARCH}" ;; 452 amd64) subdir="x86_64" ;; 453 arm64) subdir="aarch64" ;; 454 aarch64) subdir="${ARCH}" ;; 455 *) 456 echo "${ARCH} is not supported" 457 exit 15 458 esac 459 460 inlined_policy_list="\ 461 jail/seccomp/$subdir/common_device.policy \ 462 jail/seccomp/$subdir/gpu_common.policy \ 463 jail/seccomp/$subdir/serial.policy \ 464 jail/seccomp/$subdir/net.policy \ 465 jail/seccomp/$subdir/block.policy \ 466 jail/seccomp/$subdir/vvu.policy \ 467 jail/seccomp/$subdir/vhost_user.policy \ 468 jail/seccomp/$subdir/vhost_vsock.policy" 469 for policy_file in "jail/seccomp/$subdir/"*.policy; do 470 [[ "$inlined_policy_list" = *"$policy_file"* ]] && continue 471 jail/seccomp/policy-inliner.sh $inlined_policy_list <"$policy_file" | \ 472 grep -ve "^@frequency" \ 473 >"${OUTPUT_SECCOMP_DIR}"/$(basename "$policy_file") 474 done 475} 476 477# b/277618912: glibc's aarch64 memcpy uses unaligned accesses which seems to 478# cause SIGBUS errors on some Nvidia GPUs. 479compile_mem_routine_overrides() { 480 if [ "${ARCH}" = "aarch64" ]; then 481 echo "Compiling aligned mem routine overrides for ${ARCH}..." 482 483 mkdir -p "${SOURCE_DIR}/mem_override" 484 cd "${SOURCE_DIR}/mem_override" 485 486 git clone https://github.com/ARM-software/optimized-routines.git 487 488 cat >> mem_overrides.c <<EOF 489#include <string.h> 490#include "string/include/stringlib.h" 491void *memcpy(void *dest, const void *src, size_t n) 492{ 493 return __memcpy_aarch64(dest, src, n); 494} 495void *memmove(void *dest, const void *src, size_t n) 496{ 497 return __memmove_aarch64(dest, src, n); 498} 499EOF 500 501 gcc -Ioptimized-routines -Wl,-no-undefined -O2 -shared -fPIC -o libmem_overrides.so mem_overrides.c optimized-routines/string/aarch64/memcpy.S 502 503 cp libmem_overrides.so "${OUTPUT_BIN_DIR}" 504 else 505 echo "Skipping compiling mem routing override for ${ARCH}." 506 fi 507} 508 509compile() { 510 echo "Compiling..." 511 mkdir -p \ 512 "${WORKING_DIR}" \ 513 "${WORKING_DIR}/usr/lib" \ 514 "${OUTPUT_DIR}" \ 515 "${OUTPUT_BIN_DIR}" \ 516 "${OUTPUT_ETC_DIR}" \ 517 "${OUTPUT_SECCOMP_DIR}" 518 519 if [[ $BUILD_CROSVM -eq 1 ]]; then 520 compile_libdrm 521 compile_minijail 522 compile_minigbm 523 compile_epoxy 524 compile_virglrenderer 525 compile_libffi # wayland depends on it 526 compile_wayland 527 fi 528 529 if [[ $BUILD_GFXSTREAM -eq 1 ]]; then 530 compile_gfxstream 531 fi 532 533 compile_crosvm 534 compile_crosvm_seccomp 535 536 if [[ $BUILD_SWIFTSHADER -eq 1 ]]; then 537 compile_swiftshader 538 fi 539 540 compile_mem_routine_overrides 541 542 dpkg-query -W > "${OUTPUT_DIR}/builder-packages.txt" 543 echo "Results in ${OUTPUT_DIR}" 544} 545 546aarch64_retry() { 547 BUILD_CROSVM=1 BUILD_GFXSTREAM=1 BUILD_SWIFTSHADER=1 compile 548} 549 550aarch64_build() { 551 rm -rf "${WORKING_DIR}/*" 552 aarch64_retry 553} 554 555x86_64_retry() { 556 MINIGBM_DRV="I915 RADEON VC4" BUILD_CROSVM=1 BUILD_GFXSTREAM=1 BUILD_SWIFTSHADER=1 compile 557} 558 559x86_64_build() { 560 rm -rf "${WORKING_DIR}/*" 561 x86_64_retry 562} 563 564if [[ $# -lt 1 ]]; then 565 echo Choosing default config 566 set setup_env prepare_source x86_64_build 567fi 568 569echo Steps: "$@" 570 571for i in "$@"; do 572 echo $i 573 case "$i" in 574 ARCH=*) ARCH="${i/ARCH=/}" ;; 575 CUSTOM_MANIFEST=*) CUSTOM_MANIFEST="${i/CUSTOM_MANIFEST=/}" ;; 576 aarch64_build) $i ;; 577 aarch64_retry) $i ;; 578 setup_env) $i ;; 579 install_packages) $i ;; 580 fetch_source) $i ;; 581 resync_source) $i ;; 582 prepare_source) $i ;; 583 x86_64_build) $i ;; 584 x86_64_retry) $i ;; 585 *) echo $i unknown 1>&2 586 echo usage: $0 'install_packages|prepare_source|resync_source|fetch_source|$(uname -m)_build|$(uname -m)_retry' 1>&2 587 exit 2 588 ;; 589 esac 590done 591