1#!/usr/bin/env bash 2# shellcheck disable=SC1091 # The relative paths in this file only become valid at runtime. 3# shellcheck disable=SC2034 # Variables are used in scripts called from here 4# shellcheck disable=SC2086 # we want word splitting 5# shellcheck disable=SC2016 # non-expanded variables are intentional 6# When changing this file, you need to bump the following 7# .gitlab-ci/image-tags.yml tags: 8# KERNEL_ROOTFS_TAG 9# If you need to update the fluster vectors cache without updating the fluster revision, 10# you can update the FLUSTER_VECTORS_VERSION tag in .gitlab-ci/image-tags.yml. 11# When changing FLUSTER_REVISION, KERNEL_ROOTFS_TAG needs to be updated as well to rebuild 12# the rootfs. 13 14set -e 15 16. .gitlab-ci/setup-test-env.sh 17 18set -o xtrace 19 20export DEBIAN_FRONTEND=noninteractive 21: "${LLVM_VERSION:?llvm version not set!}" 22export FIRMWARE_FILES="${FIRMWARE_FILES}" 23export SKIP_UPDATE_FLUSTER_VECTORS=0 24 25check_minio() 26{ 27 S3_PATH="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}" 28 if curl -L --retry 4 -f --retry-delay 60 -s -X HEAD \ 29 "https://${S3_PATH}/done"; then 30 echo "Remote files are up-to-date, skip rebuilding them." 31 exit 32 fi 33} 34 35check_fluster() 36{ 37 S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" 38 if curl -L --retry 4 -f --retry-delay 60 -s -X HEAD \ 39 "https://${S3_PATH_FLUSTER}/done"; then 40 echo "Fluster vectors are up-to-date, skip downloading them." 41 export SKIP_UPDATE_FLUSTER_VECTORS=1 42 fi 43} 44 45check_minio "${FDO_UPSTREAM_REPO}" 46check_minio "${CI_PROJECT_PATH}" 47 48check_fluster "${FDO_UPSTREAM_REPO}" 49check_fluster "${CI_PROJECT_PATH}" 50 51. .gitlab-ci/container/container_pre_build.sh 52 53# Install rust, which we'll be using for deqp-runner. It will be cleaned up at the end. 54. .gitlab-ci/container/build-rust.sh 55 56if [[ "$DEBIAN_ARCH" = "arm64" ]]; then 57 BUILD_CL="ON" 58 BUILD_VK="ON" 59 GCC_ARCH="aarch64-linux-gnu" 60 KERNEL_ARCH="arm64" 61 DEFCONFIG="arch/arm64/configs/defconfig" 62 DEVICE_TREES="rk3399-gru-kevin.dtb" 63 DEVICE_TREES+=" meson-g12b-a311d-khadas-vim3.dtb" 64 DEVICE_TREES+=" meson-gxl-s805x-libretech-ac.dtb" 65 DEVICE_TREES+=" meson-gxm-khadas-vim2.dtb" 66 DEVICE_TREES+=" sun50i-h6-pine-h64.dtb" 67 DEVICE_TREES+=" imx8mq-nitrogen.dtb" 68 DEVICE_TREES+=" mt8192-asurada-spherion-r0.dtb" 69 DEVICE_TREES+=" mt8183-kukui-jacuzzi-juniper-sku16.dtb" 70 DEVICE_TREES+=" tegra210-p3450-0000.dtb" 71 DEVICE_TREES+=" apq8016-sbc-usb-host.dtb" 72 DEVICE_TREES+=" apq8096-db820c.dtb" 73 DEVICE_TREES+=" sc7180-trogdor-lazor-limozeen-nots-r5.dtb" 74 DEVICE_TREES+=" sc7180-trogdor-kingoftown.dtb" 75 DEVICE_TREES+=" sm8350-hdk.dtb" 76 KERNEL_IMAGE_NAME="Image" 77 78elif [[ "$DEBIAN_ARCH" = "armhf" ]]; then 79 BUILD_CL="OFF" 80 BUILD_VK="OFF" 81 GCC_ARCH="arm-linux-gnueabihf" 82 KERNEL_ARCH="arm" 83 DEFCONFIG="arch/arm/configs/multi_v7_defconfig" 84 DEVICE_TREES="rk3288-veyron-jaq.dtb" 85 DEVICE_TREES+=" sun8i-h3-libretech-all-h3-cc.dtb" 86 DEVICE_TREES+=" imx6q-cubox-i.dtb" 87 DEVICE_TREES+=" tegra124-jetson-tk1.dtb" 88 KERNEL_IMAGE_NAME="zImage" 89 . .gitlab-ci/container/create-cross-file.sh armhf 90 CONTAINER_ARCH_PACKAGES=( 91 libegl1-mesa-dev:armhf 92 libelf-dev:armhf 93 libgbm-dev:armhf 94 libgles2-mesa-dev:armhf 95 libpng-dev:armhf 96 libudev-dev:armhf 97 libvulkan-dev:armhf 98 libwaffle-dev:armhf 99 libwayland-dev:armhf 100 libx11-xcb-dev:armhf 101 libxkbcommon-dev:armhf 102 ) 103else 104 BUILD_CL="ON" 105 BUILD_VK="ON" 106 GCC_ARCH="x86_64-linux-gnu" 107 KERNEL_ARCH="x86_64" 108 DEFCONFIG="arch/x86/configs/x86_64_defconfig" 109 DEVICE_TREES="" 110 KERNEL_IMAGE_NAME="bzImage" 111 CONTAINER_ARCH_PACKAGES=( 112 libasound2-dev libcap-dev libfdt-dev libva-dev p7zip wine 113 ) 114fi 115 116# Determine if we're in a cross build. 117if [[ -e /cross_file-$DEBIAN_ARCH.txt ]]; then 118 EXTRA_MESON_ARGS="--cross-file /cross_file-$DEBIAN_ARCH.txt" 119 EXTRA_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=/toolchain-$DEBIAN_ARCH.cmake" 120 121 if [ $DEBIAN_ARCH = arm64 ]; then 122 RUST_TARGET="aarch64-unknown-linux-gnu" 123 elif [ $DEBIAN_ARCH = armhf ]; then 124 RUST_TARGET="armv7-unknown-linux-gnueabihf" 125 fi 126 rustup target add $RUST_TARGET 127 export EXTRA_CARGO_ARGS="--target $RUST_TARGET" 128 129 export ARCH=${KERNEL_ARCH} 130 export CROSS_COMPILE="${GCC_ARCH}-" 131fi 132 133# no need to remove these at end, image isn't saved at the end 134CONTAINER_EPHEMERAL=( 135 arch-test 136 automake 137 bc 138 "clang-${LLVM_VERSION}" 139 cmake 140 curl 141 mmdebstrap 142 git 143 glslang-tools 144 jq 145 libdrm-dev 146 libegl1-mesa-dev 147 libxext-dev 148 libfontconfig-dev 149 libgbm-dev 150 libgl-dev 151 libgles2-mesa-dev 152 libglu1-mesa-dev 153 libglx-dev 154 libpng-dev 155 libssl-dev 156 libudev-dev 157 libvulkan-dev 158 libwaffle-dev 159 libwayland-dev 160 libx11-xcb-dev 161 libxcb-dri2-0-dev 162 libxkbcommon-dev 163 libwayland-dev 164 ninja-build 165 openssh-server 166 patch 167 protobuf-compiler 168 python-is-python3 169 python3-distutils 170 python3-mako 171 python3-numpy 172 python3-serial 173 python3-venv 174 unzip 175 wayland-protocols 176 zstd 177) 178 179[ "$BUILD_CL" == "ON" ] && CONTAINER_EPHEMERAL+=( 180 ocl-icd-opencl-dev 181) 182 183 184echo "deb [trusted=yes] https://gitlab.freedesktop.org/gfx-ci/ci-deb-repo/-/raw/${PKG_REPO_REV}/ ${FDO_DISTRIBUTION_VERSION%-*} main" | tee /etc/apt/sources.list.d/gfx-ci_.list 185 186. .gitlab-ci/container/debian/maybe-add-llvm-repo.sh 187 188apt-get update 189apt-get install -y --no-remove \ 190 -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' \ 191 "${CONTAINER_EPHEMERAL[@]}" \ 192 "${CONTAINER_ARCH_PACKAGES[@]}" \ 193 ${EXTRA_LOCAL_PACKAGES} 194 195export ROOTFS=/lava-files/rootfs-${DEBIAN_ARCH} 196mkdir -p "$ROOTFS" 197 198# rootfs packages 199PKG_BASE=( 200 tzdata mount 201) 202PKG_CI=( 203 firmware-realtek 204 bash ca-certificates curl 205 initramfs-tools jq netcat-openbsd dropbear openssh-server 206 libasan8 207 libubsan1 208 git 209 python3-dev python3-pip python3-setuptools python3-wheel 210 weston # Wayland 211 xinit xserver-xorg-core xwayland # X11 212) 213PKG_MESA_DEP=( 214 libdrm2 libsensors5 libexpat1 # common 215 libvulkan1 # vulkan 216 libx11-6 libx11-xcb1 libxcb-dri2-0 libxcb-dri3-0 libxcb-glx0 libxcb-present0 libxcb-randr0 libxcb-shm0 libxcb-sync1 libxcb-xfixes0 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrender1 libxshmfence1 libxxf86vm1 # X11 217) 218PKG_DEP=( 219 libpng16-16 220 libva-wayland2 221 libwaffle-1-0 222 libpython3.11 python3 python3-lxml python3-mako python3-numpy python3-packaging python3-pil python3-renderdoc python3-requests python3-simplejson python3-yaml # Python 223 sntp 224 strace 225 waffle-utils 226 zstd 227) 228# arch dependent rootfs packages 229[ "$DEBIAN_ARCH" = "arm64" ] && PKG_ARCH=( 230 libgl1 libglu1-mesa 231 firmware-linux-nonfree firmware-qcom-media 232 libfontconfig1 233) 234[ "$DEBIAN_ARCH" = "amd64" ] && PKG_ARCH=( 235 firmware-amd-graphics 236 firmware-misc-nonfree 237 gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-tools gstreamer1.0-vaapi libgstreamer1.0-0 # Fluster 238 libgl1 libglu1-mesa 239 inetutils-syslogd iptables libcap2 240 libfontconfig1 241 spirv-tools 242 libelf1 libfdt1 "libllvm${LLVM_VERSION}" 243 libva2 libva-drm2 244 socat 245 sysvinit-core 246 wine 247) 248[ "$DEBIAN_ARCH" = "armhf" ] && PKG_ARCH=( 249 firmware-misc-nonfree 250) 251 252[ "$BUILD_CL" == "ON" ] && PKG_ARCH+=( 253 clinfo 254 "libclang-cpp${LLVM_VERSION}" 255 "libclang-common-${LLVM_VERSION}-dev" 256 ocl-icd-libopencl1 257) 258[ "$BUILD_VK" == "ON" ] && PKG_ARCH+=( 259 libvulkan-dev 260) 261 262mmdebstrap \ 263 --variant=apt \ 264 --arch="${DEBIAN_ARCH}" \ 265 --components main,contrib,non-free-firmware \ 266 --customize-hook='.gitlab-ci/container/get-firmware-from-source.sh "$ROOTFS" "$FIRMWARE_FILES"' \ 267 --include "${PKG_BASE[*]} ${PKG_CI[*]} ${PKG_DEP[*]} ${PKG_MESA_DEP[*]} ${PKG_ARCH[*]}" \ 268 bookworm \ 269 "$ROOTFS/" \ 270 "http://deb.debian.org/debian" \ 271 "deb [trusted=yes] https://gitlab.freedesktop.org/gfx-ci/ci-deb-repo/-/raw/${PKG_REPO_REV}/ ${FDO_DISTRIBUTION_VERSION%-*} main" \ 272 "${LLVM_APT_REPO:-}" 273 274############### Install mold 275. .gitlab-ci/container/build-mold.sh 276 277############### Building 278STRIP_CMD="${GCC_ARCH}-strip" 279mkdir -p $ROOTFS/usr/lib/$GCC_ARCH 280 281############### Build libclc 282 283if [ "$BUILD_CL" = "ON" ]; then 284 rm -rf /usr/lib/clc/* 285 . .gitlab-ci/container/build-libclc.sh 286 mkdir -p $ROOTFS/usr/{share,lib}/clc 287 mv /usr/share/clc/spirv*-mesa3d-.spv $ROOTFS/usr/share/clc/ 288 ln -s /usr/share/clc/spirv64-mesa3d-.spv $ROOTFS/usr/lib/clc/ 289 ln -s /usr/share/clc/spirv-mesa3d-.spv $ROOTFS/usr/lib/clc/ 290fi 291 292############### Build Vulkan validation layer (for zink) 293if [ "$DEBIAN_ARCH" = "amd64" ]; then 294 . .gitlab-ci/container/build-vulkan-validation.sh 295 mv /usr/lib/x86_64-linux-gnu/libVkLayer_khronos_validation.so $ROOTFS/usr/lib/x86_64-linux-gnu/ 296 mkdir -p $ROOTFS/usr/share/vulkan/explicit_layer.d 297 mv /usr/share/vulkan/explicit_layer.d/* $ROOTFS/usr/share/vulkan/explicit_layer.d/ 298fi 299 300############### Build apitrace 301. .gitlab-ci/container/build-apitrace.sh 302mkdir -p $ROOTFS/apitrace 303mv /apitrace/build $ROOTFS/apitrace 304rm -rf /apitrace 305 306############### Build ANGLE 307if [[ "$DEBIAN_ARCH" = "amd64" ]]; then 308 . .gitlab-ci/container/build-angle.sh 309 mv /angle $ROOTFS/. 310 rm -rf /angle 311fi 312 313############### Build dEQP runner 314. .gitlab-ci/container/build-deqp-runner.sh 315mkdir -p $ROOTFS/usr/bin 316mv /usr/local/bin/*-runner $ROOTFS/usr/bin/. 317 318 319############### Build dEQP 320 321DEQP_API=tools \ 322DEQP_TARGET=default \ 323. .gitlab-ci/container/build-deqp.sh 324 325DEQP_API=GL \ 326DEQP_TARGET=surfaceless \ 327. .gitlab-ci/container/build-deqp.sh 328 329DEQP_API=GLES \ 330DEQP_TARGET=surfaceless \ 331. .gitlab-ci/container/build-deqp.sh 332 333if [ "$BUILD_VK" == "ON" ]; then 334 DEQP_API=VK \ 335 DEQP_TARGET=default \ 336 . .gitlab-ci/container/build-deqp.sh 337 338 DEQP_API=VK-main \ 339 DEQP_TARGET=default \ 340 . .gitlab-ci/container/build-deqp.sh 341fi 342 343rm -rf /VK-GL-CTS 344 345mv /deqp-* $ROOTFS/. 346 347 348############### Build SKQP 349if [[ "$DEBIAN_ARCH" = "arm64" ]] \ 350 || [[ "$DEBIAN_ARCH" = "amd64" ]]; then 351 . .gitlab-ci/container/build-skqp.sh 352 mv /skqp $ROOTFS/. 353fi 354 355############### Build piglit 356PIGLIT_OPTS="-DPIGLIT_USE_WAFFLE=ON 357 -DPIGLIT_USE_GBM=ON 358 -DPIGLIT_USE_WAYLAND=ON 359 -DPIGLIT_USE_X11=ON 360 -DPIGLIT_BUILD_GLX_TESTS=ON 361 -DPIGLIT_BUILD_EGL_TESTS=ON 362 -DPIGLIT_BUILD_WGL_TESTS=OFF 363 -DPIGLIT_BUILD_GL_TESTS=ON 364 -DPIGLIT_BUILD_GLES1_TESTS=ON 365 -DPIGLIT_BUILD_GLES2_TESTS=ON 366 -DPIGLIT_BUILD_GLES3_TESTS=ON 367 -DPIGLIT_BUILD_CL_TESTS=$BUILD_CL 368 -DPIGLIT_BUILD_VK_TESTS=$BUILD_VK 369 -DPIGLIT_BUILD_DMA_BUF_TESTS=ON" \ 370 . .gitlab-ci/container/build-piglit.sh 371mv /piglit $ROOTFS/. 372 373############### Build libva tests 374if [[ "$DEBIAN_ARCH" = "amd64" ]]; then 375 . .gitlab-ci/container/build-va-tools.sh 376 mv /va/bin/* $ROOTFS/usr/bin/ 377fi 378 379############### Build Crosvm 380if [[ ${DEBIAN_ARCH} = "amd64" ]]; then 381 . .gitlab-ci/container/build-crosvm.sh 382 mv /usr/local/bin/crosvm $ROOTFS/usr/bin/ 383 mv /usr/local/lib/libvirglrenderer.* $ROOTFS/usr/lib/$GCC_ARCH/ 384 mkdir -p $ROOTFS/usr/local/libexec/ 385 mv /usr/local/libexec/virgl* $ROOTFS/usr/local/libexec/ 386fi 387 388############### Build ci-kdl 389. .gitlab-ci/container/build-kdl.sh 390mv /ci-kdl $ROOTFS/ 391 392############### Install fluster 393if [[ ${DEBIAN_ARCH} = "amd64" ]]; then 394 section_start fluster "Install fluster" 395 . .gitlab-ci/container/build-fluster.sh 396 section_end fluster 397fi 398 399############### Build local stuff for use by igt and kernel testing, which 400############### will reuse most of our container build process from a specific 401############### hash of the Mesa tree. 402if [[ -e ".gitlab-ci/local/build-rootfs.sh" ]]; then 403 . .gitlab-ci/local/build-rootfs.sh 404fi 405 406 407############### Download prebuilt kernel 408. .gitlab-ci/container/download-prebuilt-kernel.sh 409 410############### Delete rust, since the tests won't be compiling anything. 411rm -rf /root/.cargo 412rm -rf /root/.rustup 413 414############### Delete firmware files we don't need 415if [ "$DEBIAN_ARCH" = "amd64" ]; then 416 dpkg -L firmware-misc-nonfree | grep -v "i915" | xargs rm || true 417fi 418 419############### Fill rootfs 420cp .gitlab-ci/setup-test-env.sh $ROOTFS/. 421cp .gitlab-ci/container/setup-rootfs.sh $ROOTFS/. 422cp .gitlab-ci/container/strip-rootfs.sh $ROOTFS/. 423cp .gitlab-ci/container/debian/llvm-snapshot.gpg.key $ROOTFS/. 424cp .gitlab-ci/container/debian/winehq.gpg.key $ROOTFS/. 425chroot $ROOTFS bash /setup-rootfs.sh 426rm $ROOTFS/{llvm-snapshot,winehq}.gpg.key 427rm "$ROOTFS/setup-test-env.sh" 428rm "$ROOTFS/setup-rootfs.sh" 429rm "$ROOTFS/strip-rootfs.sh" 430cp /etc/wgetrc $ROOTFS/etc/. 431 432if [ "${DEBIAN_ARCH}" = "arm64" ]; then 433 mkdir -p /lava-files/rootfs-arm64/lib/firmware/qcom/sm8350/ # for firmware imported later 434 # Make a gzipped copy of the Image for db410c. 435 gzip -k /lava-files/Image 436 KERNEL_IMAGE_NAME+=" Image.gz" 437fi 438 439ROOTFSTAR="lava-rootfs.tar.zst" 440du -ah "$ROOTFS" | sort -h | tail -100 441pushd $ROOTFS 442 tar --zstd -cf /lava-files/${ROOTFSTAR} . 443popd 444 445. .gitlab-ci/container/container_post_build.sh 446 447ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/"${ROOTFSTAR}" \ 448 https://${S3_PATH}/"${ROOTFSTAR}" 449 450touch /lava-files/done 451ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done https://${S3_PATH}/done 452