1#!/usr/bin/env bash 2# Copyright (c) 2021-2024 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# 16# Aux functions 17# 18 19function build_and_install_vmb 20{ 21 make -C "${SCRIPT_DIR}/../tests/vm-benchmarks/" vmb 22 23} 24 25function enable_llvm_repo 26{ 27 set -x 28 local llvm_url=${LLVM_REPO:-'https://apt.llvm.org'} 29 local llvm_version=$1 30 local llvm_gpg_url=${LLVM_GPG_REPO:-'https://apt.llvm.org'} 31 32 local repo_name="deb ${llvm_url}/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${llvm_version} main" 33 curl --retry 5 --retry-delay 10 -k ${llvm_gpg_url}/llvm-snapshot.gpg.key | apt-key add - 34 echo -e $repo_name > /etc/apt/sources.list.d/llvm-${llvm_version}.list 35 apt-get update 36 set +x 37} 38 39function unpack_clang_rt 40{ 41 local llvm_version=$1 42 43 # install requirements 44 apt-get update 45 apt-get install -y --no-install-recommends \ 46 binutils \ 47 gzip \ 48 tar \ 49 xz-utils 50 mkdir -p clang_rt && cd clang_rt 51 apt-get download libclang-rt-${llvm_version}-dev 52 ar x libclang-rt-${llvm_version}-dev*.deb 53 tar xJf data.tar.xz 54 cp -r usr / 55 cd ../ 56 rm -rf clang_rt 57} 58 59SCRIPT_DIR="$(realpath "${0}")" 60SCRIPT_DIR="$(dirname "${SCRIPT_DIR}")" 61cd "${SCRIPT_DIR}" 62 63if [[ -f "${SCRIPT_DIR}/extras/install-deps-extras.sh" ]] 64then 65 source "${SCRIPT_DIR}/extras/install-deps-extras.sh" 66fi 67 68function print_help 69{ 70 HELP_MESSAGE=" 71 It is the bootstrap script for Panda on Ubuntu 18.04 or 20.04. 72 73 This script installs all necessary packages for building and testing Panda 74 in your local environment, given that your environment is Ubuntu 18.04 or 20.04. 75 (detected with the contents of /etc/os-release). 76 77 The script should run with superuser privileges. 78 79 EXAMPLE 80 81 $ ./scripts/install-deps-ubuntu --help 82 $ ./scripts/install-deps-ubuntu --install=x86 --install=arm-all --install=dev 83 84 or 85 86 $ ./scripts/install-deps-ubuntu -h 87 $ ./scripts/install-deps-ubuntu -i=x86 -i=arm-all -i=dev 88 89 SYNOPSIS 90 91 $0 [OPTIONS] 92 93 OPTIONS 94 95 --help | -h Show this message and exit 96 97 --install=dev | -i=dev Install tools needed for development 98 99 --install=arm-dev | -i=arm-dev Install ARM64-hosted tools needed for development 100 101 --install=arm-all | -i=arm-all Install extra packages for cross-compiling for AArch32 and AArch64 102 103 --install=x86 | -i=x86 Install extra packages for cross-compiling for x86 104 105 --install=windows | -i=windows Install extra packages for cross-compiling for Windows 106 107 --install=doc-dev | -i=doc-dev Install tools for documentation development 108 109 --install=test | -i=test Install python dependencies for tests running 110 111 --install=vmb | -i=vmb Installs VMB framework to run benchmarks from CLI 112 " 113 114 if [[ -n "${EXTRA_OPTIONS}" ]] 115 then 116 HELP_MESSAGE="${HELP_MESSAGE}${ADDITIONAL_OPTIONS_HELP}" 117 fi 118 119 120 HELP_MESSAGE="${HELP_MESSAGE} 121 CAVEAT 122 123 * Packages for cross-compiling for aarch64 and x86 cannot co-exist, so the 124 script (read: apt) will replace any conflicting packages on each run. 125 * However, packages for cross-compiling for aarch64 and 32-bit ARM can 126 co-exist, so they are in a single 'arm-all' dependency list. 127 " 128 129 echo "$HELP_MESSAGE" 130} 131 132function install_dep 133{ 134 local fname=$1 135 136 if [[ ! -f "$fname" ]] ; then 137 echo "FATAL: Dependency list $fname not found." 138 exit 1 139 fi 140 141 echo "Processing $fname" 142 grep --color=never -o '^[^#]*' "$fname" | xargs apt install -y --no-install-recommends -o Dpkg::Options::="--force-overwrite" -o Acquire::Retries=30 -o Acquire::https::Timeout=600 -o Acquire::http::Timeout=600 143} 144 145function install_pip_dep 146{ 147 local fname=$1 148 149 if [[ ! -f "$fname" ]] ; then 150 echo "FATAL: Dependency list $fname not found." 151 exit 1 152 fi 153 154 echo "Processing $fname" 155 python3 -m pip install --no-cache-dir --upgrade -r $fname 156} 157 158function enable_test_toolchain_repo 159{ 160 set -x 161 local repo_url=${PPA_REPO:-'https://ppa.launchpadcontent.net/ubuntu-toolchain-r/test/ubuntu'} 162 163 local repo_name="deb ${repo_url} ${UBUNTU_CODENAME} main" 164 curl --retry 5 --retry-delay 10 -k "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x60C317803A41BA51845E371A1E9377A2BA9EF27F" | apt-key add - 165 echo -e $repo_name > /etc/apt/sources.list.d/ubuntu-toolchain-r-test.list 166 167 cat > /etc/apt/preferences.d/ubuntu-toolchain-r-test << EOF 168Package: * 169Pin: release o=LP-PPA-ubuntu-toolchain-r-test 170Pin-Priority: 1 171EOF 172 173 apt-get update 174 set +x 175} 176 177function install_cross_zlib 178{ 179 local zlib_url=${ZLIB_URL:-'https://zlib.net/fossils/zlib-1.2.11.tar.gz'} 180 TEMP=`mktemp -d` 181 pushd . 182 cd $TEMP 183 curl --retry 5 --retry-delay 10 -o zlib.tar.gz "${zlib_url}" 184 tar xf zlib.tar.gz 185 cd zlib-1.2.11 186 187 if [ `which aarch64-linux-gnu-gcc-8` ]; then 188 CC=aarch64-linux-gnu-gcc-8 ./configure --prefix=/usr/aarch64-linux-gnu 189 make -j install 190 make clean 191 fi 192 193 if [ `which arm-linux-gnueabi-gcc-8` ]; then 194 CC=arm-linux-gnueabi-gcc-8 ./configure --prefix=/usr/arm-linux-gnueabi 195 make -j install 196 make clean 197 fi 198 199 if [ `which arm-linux-gnueabihf-gcc-8` ]; then 200 CC=arm-linux-gnueabihf-gcc-8 ./configure --prefix=/usr/arm-linux-gnueabihf 201 make -j install 202 make clean 203 fi 204 205 popd 206 rm -rf $TEMP 207} 208 209function install_cross_libdwarf 210{ 211 local lib_url=${LIBDWRF_URL:-'https://codeload.github.com/davea42/libdwarf-code/tar.gz/refs/tags/20201020'} 212 TEMP=`mktemp -d` 213 pushd . 214 cd $TEMP 215 curl --retry 5 --retry-delay 10 -o libdwarf.tar.gz "${lib_url}" 216 tar xf libdwarf.tar.gz 217 cd libdwarf-code-20201020 218 autoreconf 219 220 if [ `which aarch64-linux-gnu-gcc-8` ]; then 221 CC=aarch64-linux-gnu-gcc-8 ./configure --host=aarch64-linux-gnu --prefix=/usr/aarch64-linux-gnu --includedir=/usr/aarch64-linux-gnu/include/libdwarf --disable-libelf --enable-shared --with-pic 222 make -j install 223 make clean 224 fi 225 226 if [ `which arm-linux-gnueabi-gcc-8` ]; then 227 CC=arm-linux-gnueabi-gcc-8 ./configure --host=arm-linux-gnueabi --prefix=/usr/arm-linux-gnueabi --includedir=/usr/arm-linux-gnueabi/include/libdwarf --disable-libelf --enable-shared --with-pic 228 make -j install 229 make clean 230 fi 231 232 if [ `which arm-linux-gnueabihf-gcc-8` ]; then 233 CC=arm-linux-gnueabihf-gcc-8 ./configure --host=arm-linux-gnueabihf --prefix=/usr/arm-linux-gnueabihf --includedir=/usr/arm-linux-gnueabihf/include/libdwarf --disable-libelf --enable-shared --with-pic 234 make -j install 235 make clean 236 fi 237 238 popd 239 rm -rf $TEMP 240} 241 242function install_cross_libs 243{ 244 echo "Installing libraries for cross-compilation ..." 245 apt install -y automake 246 install_cross_zlib 247 install_cross_libdwarf 248 echo "Success" 249} 250 251function install_python_dependencies 252{ 253 if [[ $VERSION_ID == "18.04" ]]; then 254 echo "Install python 3.8" 255 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 256 fi 257 258 local REQUIREMENTS_BASE_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-base-python3 259 local REQUIREMENTS_ALL_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-python3 260 local REQUIREMENTS_VENV_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-venv-python3 261 262 echo "Installing pip packages for all" 263 install_pip_dep "${REQUIREMENTS_BASE_PYTHON3_NAME}" 264 install_pip_dep "${REQUIREMENTS_ALL_PYTHON3_NAME}" 265 266 echo "Creating venv and installing pip packages there" 267 local MY_USERNAME=${SUDO_USER} 268 if [[ -z "${VENV_DIR}" && -n "${MY_USERNAME}" ]]; then 269 local MY_HOME=$(grep "^${MY_USERNAME}:" /etc/passwd | cut -d: -f6) 270 if [[ ! -e "${MY_HOME}" ]]; then 271 MY_HOME=/home/${MY_USERNAME} 272 fi 273 VENV_DIR=${MY_HOME}/.venv-panda 274 elif [[ -z "${VENV_DIR}" && -z "${MY_USERNAME}" ]]; then 275 VENV_DIR=/root/.venv-panda 276 fi 277 278 echo "Going to create a virtual environment at ${VENV_DIR}" 279 python3 -m venv "${VENV_DIR}" 280 281 source "${VENV_DIR}/bin/activate" 282 install_pip_dep "${REQUIREMENTS_BASE_PYTHON3_NAME}" 283 install_pip_dep "${REQUIREMENTS_VENV_PYTHON3_NAME}" 284 deactivate 285 286 if [[ -n "${MY_USERNAME}" ]]; then 287 local MY_UID=$(id -u "${MY_USERNAME}") 288 local MY_GID=$(id -g "${MY_USERNAME}") 289 chown -R $MY_UID:$MY_GID $VENV_DIR 290 fi 291} 292 293# 294# Main logic 295# 296 297# 298# Parse command-line arguments 299# 300 301# Set default flag values 302INSTALL_DEV=no 303INSTALL_CROSS_x86=no 304INSTALL_CROSS_WINDOWS=no 305INSTALL_ARM_DEV=no 306INSTALL_CROSS_ARM_ALL=no 307INSTALL_CROSS_LIBS=no 308INSTALL_DOC_DEV=no 309INSTALL_TEST=no 310INSTALL_VMB=no 311ADD_TOOLCHAIN_REPOS=no 312SRC_LIST_STR='# This file is generated automatically by Panda install-deps-ubuntu script. DO NOT EDIT!!!\n' 313 314for i in "$@" 315do 316 ERROR_ARG="" 317 case $i in 318 -h|--help) 319 print_help 320 exit 0 321 ;; 322 -i=*|--install=*) 323 FLAG_ARG=${i//[-a-zA-Z0-9]*=/} 324 if [[ $FLAG_ARG == "dev" ]] ; then 325 if [[ $INSTALL_ARM_DEV == "yes" ]] ; then 326 echo "FATAL: Parameter --install=dev excludes --install=arm-dev" 327 exit 1 328 else 329 INSTALL_DEV=yes 330 fi 331 fi 332 if [[ $FLAG_ARG == "x86" ]] ; then 333 INSTALL_CROSS_x86=yes 334 fi 335 if [[ $FLAG_ARG == "arm-all" ]] ; then 336 INSTALL_CROSS_ARM_ALL=yes 337 INSTALL_CROSS_LIBS=yes 338 fi 339 if [[ $FLAG_ARG == "windows" ]] ; then 340 INSTALL_CROSS_WINDOWS=yes 341 fi 342 if [[ $FLAG_ARG == "arm-dev" ]] ; then 343 if [[ $INSTALL_DEV == "yes" ]] ; then 344 echo "FATAL: Parameter --install=arm-dev excludes --install=dev" 345 exit 1 346 else 347 INSTALL_ARM_DEV=yes 348 fi 349 fi 350 if [[ $FLAG_ARG == "doc-dev" ]] ; then 351 INSTALL_DOC_DEV=yes 352 fi 353 if [[ $FLAG_ARG == "test" ]] ; then 354 INSTALL_TEST=yes 355 fi 356 if [[ $FLAG_ARG == "vmb" ]] ; then 357 INSTALL_VMB=yes 358 fi 359 ;; 360 *) 361 ERROR_ARG="YES" 362 ;; 363 esac 364 365 if [[ -n "${EXTRA_OPTIONS}" ]] 366 then 367 extra_parse "${i}" 368 fi 369 370 if [[ -n "${ERROR_ARG}" ]] 371 then 372 echo "Error: Unsupported flag $i" >&2 373 exit 1 374 fi 375 376done 377 378# 379# Check 'sudo' and if script is running on Ubuntu 380# 381 382if [[ $(id -u) -ne 0 ]] ; then 383 print_help 384 echo "!!!" 385 echo "FATAL: Please run as root." 386 echo "!!!" 387 exit 1 388fi 389 390# Install deps for the script itself. Until this line we assume that 391# only shell builtins and commands from required packages are invoked. 392apt-get update 393install_dep "$SCRIPT_DIR/dep-lists/ubuntu-bootstrap" 394 395# 396# Check specific Ubuntu version 397# 398 399UBUNTU_NAME=ubuntu-18-04 400 401if [ ! -f /etc/os-release ]; then 402 echo "FATAL: /etc/os-release not found. Exiting..." 403 exit 1 404else 405 . /etc/os-release 406 407 if [[ $VERSION_ID == "18.04" ]]; then 408 echo "Installing packages for Ubuntu 18.04 LTS." 409 UBUNTU_NAME=ubuntu-18-04 410 ADD_TOOLCHAIN_REPOS=yes 411 elif [[ $VERSION_ID == "20.04" ]]; then 412 echo "Installing packages for Ubuntu 20.04 LTS." 413 UBUNTU_NAME=ubuntu-20-04 414 ADD_TOOLCHAIN_REPOS=yes 415 elif [[ $VERSION_ID == "22.04" ]]; then 416 echo "Installing packages for Ubuntu 22.04 LTS" 417 UBUNTU_NAME=ubuntu-22-04 418 else 419 echo "Trying to install packages for Ubuntu with unpinned versions." 420 fi 421 422 if [[ $NAME == "Ubuntu" ]] 423 then 424 set -x 425 apt-get update 426 dpkg -l | grep curl || apt-get -y install curl 427 dpkg -l | grep gnupg || apt-get -y install gnupg 428 429 if [[ -n "${EXTRA_OPTIONS}" ]] 430 then 431 extra_add_repo 432 fi 433 434 set +x 435 else 436 echo "FATAL: Only Ubuntu is supported. This is not. Exiting..." 437 exit 1 438 fi 439fi 440 441set -e 442 443# 444# Install dependencies 445# 446 447if [[ "x$ADD_TOOLCHAIN_REPOS" == "xyes" ]] ; then 448 enable_llvm_repo 14 449 if [[ $VERSION_ID == "18.04" ]]; then 450 enable_test_toolchain_repo 451 fi 452 unpack_clang_rt 14 453fi 454 455install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-build" 456 457if [[ "x$INSTALL_DEV" == "xyes" ]] ; then 458 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-dev" 459fi 460 461if [[ "x$INSTALL_ARM_DEV" == "xyes" ]] ; then 462 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-arm-dev" 463fi 464 465if [[ "x$INSTALL_CROSS_x86" == "xyes" ]] ; then 466 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-x86" 467fi 468 469if [[ "x$INSTALL_CROSS_WINDOWS" == "xyes" ]] ; then 470 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-windows" 471fi 472 473if [[ "x$INSTALL_CROSS_ARM_ALL" == "xyes" ]] ; then 474 if [[ -z "${EXTRA_OPTIONS}" ]] 475 then 476 "${SCRIPT_DIR}/install-deps-qemu" 477 fi 478 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-arm-all" 479fi 480if [[ "$INSTALL_CROSS_LIBS" == "yes" ]]; then 481 install_cross_libs 482fi 483 484if [[ "x$INSTALL_DOC_DEV" == "xyes" ]]; then 485 if [[ -f "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-doc-dev" ]] ; then 486 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-doc-dev" 487 else 488 install_dep "$SCRIPT_DIR/dep-lists/ubuntu-doc-dev" 489 fi 490 install_pip_dep "${SCRIPT_DIR}/dep-lists/requirements-doc-dev" 491fi 492 493if [[ -n "${EXTRA_OPTIONS}" ]] 494then 495 extra_install 496fi 497 498if [[ "x$INSTALL_TEST" == "xyes" ]]; then 499 install_python_dependencies 500fi 501 502 503if [[ "x$INSTALL_VMB" == "xyes" ]]; then 504 install_python_dependencies 505 install_pip_dep "${SCRIPT_DIR}/dep-lists/requirements-vmb" 506 build_and_install_vmb 507fi 508