1#!/usr/bin/env bash 2# Copyright (c) 2021-2025 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# 16# Aux functions 17# 18 19function build_and_install_vmb 20{ 21 make -C "${SCRIPT_DIR}/../tests/vm-benchmarks/" vmb 22 23} 24 25function enable_llvm_repo 26{ 27 set -x 28 local llvm_url=${LLVM_REPO:-'https://apt.llvm.org'} 29 local llvm_version=$1 30 local llvm_gpg_url=${LLVM_GPG_REPO:-'https://apt.llvm.org'} 31 32 local repo_name="deb ${llvm_url}/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${llvm_version} main" 33 curl --retry 5 --retry-delay 10 -k ${llvm_gpg_url}/llvm-snapshot.gpg.key | apt-key add - 34 echo -e $repo_name > /etc/apt/sources.list.d/llvm-${llvm_version}.list 35 apt-get update 36 set +x 37} 38 39function unpack_clang_rt 40{ 41 local llvm_version=$1 42 43 # install requirements 44 apt-get update 45 apt-get install -y --no-install-recommends \ 46 binutils \ 47 gzip \ 48 tar \ 49 xz-utils 50 mkdir -p clang_rt && cd clang_rt 51 apt-get download libclang-rt-${llvm_version}-dev 52 ar x libclang-rt-${llvm_version}-dev*.deb 53 tar xJf data.tar.xz 54 cp -r usr / 55 cd ../ 56 rm -rf clang_rt 57} 58 59SCRIPT_DIR="$(realpath "${0}")" 60SCRIPT_DIR="$(dirname "${SCRIPT_DIR}")" 61cd "${SCRIPT_DIR}" 62 63if [[ -f "${SCRIPT_DIR}/extras/install-deps-extras.sh" ]] 64then 65 source "${SCRIPT_DIR}/extras/install-deps-extras.sh" 66fi 67 68function print_help 69{ 70 HELP_MESSAGE=" 71 It is the bootstrap script for Panda on Ubuntu 18.04, 20.04, 22.04 and 24.04. 72 73 This script installs all necessary packages for building and testing Panda 74 in your local environment, given that your environment is supported Ubuntu version 75 (detected with the contents of /etc/os-release). 76 77 The script should run with superuser privileges. 78 79 EXAMPLE 80 81 $ ./scripts/install-deps-ubuntu --help 82 $ ./scripts/install-deps-ubuntu --install=x86 --install=arm-all --install=dev 83 84 or 85 86 $ ./scripts/install-deps-ubuntu -h 87 $ ./scripts/install-deps-ubuntu -i=x86 -i=arm-all -i=dev 88 89 SYNOPSIS 90 91 $0 [OPTIONS] 92 93 OPTIONS 94 95 --help | -h Show this message and exit 96 97 --install=dev | -i=dev Install tools needed for development 98 99 --install=arm-dev | -i=arm-dev Install ARM64-hosted tools needed for development 100 101 --install=arm-all | -i=arm-all Install extra packages for cross-compiling for AArch32 and AArch64 102 103 --install=x86 | -i=x86 Install extra packages for cross-compiling for x86 104 105 --install=windows | -i=windows Install extra packages for cross-compiling for Windows 106 107 --install=doc-dev | -i=doc-dev Install tools for documentation development 108 109 --install=test | -i=test Install python dependencies for tests running 110 111 --install=vmb | -i=vmb Installs VMB framework to run benchmarks from CLI 112 113 --install=llvm-prebuilts | -i=llvm-prebuilts Install deps for use llvm prebuilts. 114 115 --install=ohos-sdk | -i=ohos-sdk Install ohos SDK. 116 " 117 118 if [[ -n "${EXTRA_OPTIONS}" ]] 119 then 120 HELP_MESSAGE="${HELP_MESSAGE}${ADDITIONAL_OPTIONS_HELP}" 121 fi 122 123 124 HELP_MESSAGE="${HELP_MESSAGE} 125 CAVEAT 126 127 * Packages for cross-compiling for aarch64 and x86 cannot co-exist, so the 128 script (read: apt) will replace any conflicting packages on each run. 129 * However, packages for cross-compiling for aarch64 and 32-bit ARM can 130 co-exist, so they are in a single 'arm-all' dependency list. 131 " 132 133 echo "$HELP_MESSAGE" 134} 135 136function install_dep 137{ 138 local fname=$1 139 140 if [[ ! -f "$fname" ]] ; then 141 echo "FATAL: Dependency list $fname not found." 142 exit 1 143 fi 144 145 echo "Processing $fname" 146 grep --color=never -o '^[^#]*' "$fname" | xargs apt install -y --no-install-recommends -o Dpkg::Options::="--force-overwrite" -o Acquire::Retries=30 -o Acquire::https::Timeout=600 -o Acquire::http::Timeout=600 147} 148 149function install_pip_dep 150{ 151 local fname=$1 152 153 if [[ ! -f "$fname" ]] ; then 154 echo "FATAL: Dependency list $fname not found." 155 exit 1 156 fi 157 158 echo "Processing $fname" 159 if [[ $VERSION_ID == "24.04" ]]; then 160 python3 -m pip install --break-system-packages --no-cache-dir --upgrade -r $fname 161 else 162 python3 -m pip install --no-cache-dir --upgrade -r $fname 163 fi 164} 165 166function enable_test_toolchain_repo 167{ 168 set -x 169 local repo_url=${PPA_REPO:-'https://ppa.launchpadcontent.net/ubuntu-toolchain-r/test/ubuntu'} 170 171 local repo_name="deb ${repo_url} ${UBUNTU_CODENAME} main" 172 curl --retry 5 --retry-delay 10 -k "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x60C317803A41BA51845E371A1E9377A2BA9EF27F" | apt-key add - 173 echo -e $repo_name > /etc/apt/sources.list.d/ubuntu-toolchain-r-test.list 174 175 cat > /etc/apt/preferences.d/ubuntu-toolchain-r-test << EOF 176Package: * 177Pin: release o=LP-PPA-ubuntu-toolchain-r-test 178Pin-Priority: 1 179EOF 180 181 apt-get update 182 set +x 183} 184 185function install_cross_zlib 186{ 187 local zlib_url=${ZLIB_URL:-'https://zlib.net/fossils/zlib-1.2.11.tar.gz'} 188 TEMP=`mktemp -d` 189 pushd . 190 cd $TEMP 191 curl --retry 5 --retry-delay 10 -o zlib.tar.gz "${zlib_url}" 192 tar xf zlib.tar.gz 193 cd zlib-1.2.11 194 195 if [ `which aarch64-linux-gnu-gcc` ]; then 196 CC=aarch64-linux-gnu-gcc ./configure --prefix=/usr/aarch64-linux-gnu 197 make -j install 198 make clean 199 fi 200 201 if [ `which arm-linux-gnueabi-gcc` ]; then 202 CC=arm-linux-gnueabi-gcc ./configure --prefix=/usr/arm-linux-gnueabi 203 make -j install 204 make clean 205 fi 206 207 if [ `which arm-linux-gnueabihf-gcc` ]; then 208 CC=arm-linux-gnueabihf-gcc ./configure --prefix=/usr/arm-linux-gnueabihf 209 make -j install 210 make clean 211 fi 212 213 popd 214 rm -rf $TEMP 215} 216 217function install_cross_libdwarf 218{ 219 local lib_url=${LIBDWRF_URL:-'https://codeload.github.com/davea42/libdwarf-code/tar.gz/refs/tags/20201020'} 220 TEMP=`mktemp -d` 221 pushd . 222 cd $TEMP 223 curl --retry 5 --retry-delay 10 -o libdwarf.tar.gz "${lib_url}" 224 tar xf libdwarf.tar.gz 225 cd libdwarf-code-20201020 226 autoreconf 227 228 if [ `which aarch64-linux-gnu-gcc` ]; then 229 CC=aarch64-linux-gnu-gcc ./configure --host=aarch64-linux-gnu --prefix=/usr/aarch64-linux-gnu --includedir=/usr/aarch64-linux-gnu/include/libdwarf --disable-libelf --enable-shared --with-pic 230 make -j install 231 make clean 232 fi 233 234 if [ `which arm-linux-gnueabi-gcc` ]; then 235 CC=arm-linux-gnueabi-gcc ./configure --host=arm-linux-gnueabi --prefix=/usr/arm-linux-gnueabi --includedir=/usr/arm-linux-gnueabi/include/libdwarf --disable-libelf --enable-shared --with-pic 236 make -j install 237 make clean 238 fi 239 240 if [ `which arm-linux-gnueabihf-gcc` ]; then 241 CC=arm-linux-gnueabihf-gcc ./configure --host=arm-linux-gnueabihf --prefix=/usr/arm-linux-gnueabihf --includedir=/usr/arm-linux-gnueabihf/include/libdwarf --disable-libelf --enable-shared --with-pic 242 make -j install 243 make clean 244 fi 245 246 popd 247 rm -rf $TEMP 248} 249 250function install_cross_libs 251{ 252 echo "Installing libraries for cross-compilation ..." 253 apt install -y automake 254 install_cross_zlib 255 install_cross_libdwarf 256 echo "Success" 257} 258 259function install_python3_10_ubuntu20_04 260{ 261 echo "Installing python 3.10..." 262 apt install software-properties-common -y --no-install-recommends 263 264 local py310_repo=${PY310_REPO:-'ppa:deadsnakes/ppa'} 265 local pip_url=${PIP_REPO:-'https://bootstrap.pypa.io/get-pip.py'} 266 267 if [[ ${PY310_REPO} ]]; then 268 echo "Adding key for python3.10 repository" 269 curl --retry 5 --retry-delay 10 -k "${PY310_REPO_KEY}" | apt-key add - 270 fi 271 272 add-apt-repository "$py310_repo" 273 apt update 274 apt install -y python3.10 python3.10-venv python3.10-dev 275 echo "Installation python 3.10 complete" 276 277 echo "Set python3.10 higher priority" 278 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 279 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 280 update-alternatives --set python3 /usr/bin/python3.10 281 282 echo "Installing pip for python3.10" 283 curl --retry 5 --retry-delay 10 -sS "$pip_url" | python3.10 284 echo "pip installation complete" 285 286} 287 288function install_python_dependencies 289{ 290 291 local REQUIREMENTS_BASE_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-base-python3 292 local REQUIREMENTS_ALL_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-python3 293 local REQUIREMENTS_VENV_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-venv-python3 294 295 echo "Installing pip packages for all" 296 install_pip_dep "${REQUIREMENTS_BASE_PYTHON3_NAME}" 297 install_pip_dep "${REQUIREMENTS_ALL_PYTHON3_NAME}" 298 299 echo "Creating venv and installing pip packages there" 300 local MY_USERNAME=${SUDO_USER} 301 if [[ -z "${VENV_DIR}" && -n "${MY_USERNAME}" ]]; then 302 local MY_HOME=$(grep "^${MY_USERNAME}:" /etc/passwd | cut -d: -f6) 303 if [[ ! -e "${MY_HOME}" ]]; then 304 MY_HOME=/home/${MY_USERNAME} 305 fi 306 VENV_DIR=${MY_HOME}/.venv-panda 307 elif [[ -z "${VENV_DIR}" && -z "${MY_USERNAME}" ]]; then 308 VENV_DIR=/root/.venv-panda 309 fi 310 311 echo "Going to create a virtual environment at ${VENV_DIR}" 312 python3 -m venv "${VENV_DIR}" 313 314 source "${VENV_DIR}/bin/activate" 315 install_pip_dep "${REQUIREMENTS_BASE_PYTHON3_NAME}" 316 install_pip_dep "${REQUIREMENTS_VENV_PYTHON3_NAME}" 317 deactivate 318 319 if [[ -n "${MY_USERNAME}" ]]; then 320 local MY_UID=$(id -u "${MY_USERNAME}") 321 local MY_GID=$(id -g "${MY_USERNAME}") 322 chown -R $MY_UID:$MY_GID $VENV_DIR 323 fi 324} 325 326# 327# Main logic 328# 329 330# 331# Parse command-line arguments 332# 333 334# Set default flag values 335INSTALL_DEV=no 336INSTALL_CROSS_x86=no 337INSTALL_CROSS_WINDOWS=no 338INSTALL_ARM_DEV=no 339INSTALL_CROSS_ARM_ALL=no 340INSTALL_CROSS_LIBS=no 341INSTALL_DOC_DEV=no 342INSTALL_TEST=no 343INSTALL_VMB=no 344ADD_TOOLCHAIN_REPOS=no 345INSTALL_LLVM_PREBUILTS=no 346INSTALL_OHOS_SDK=no 347SRC_LIST_STR='# This file is generated automatically by Panda install-deps-ubuntu script. DO NOT EDIT!!!\n' 348 349for i in "$@" 350do 351 ERROR_ARG="" 352 case $i in 353 -h|--help) 354 print_help 355 exit 0 356 ;; 357 -i=*|--install=*) 358 FLAG_ARG=${i//[-a-zA-Z0-9]*=/} 359 if [[ $FLAG_ARG == "dev" ]] ; then 360 if [[ $INSTALL_ARM_DEV == "yes" ]] ; then 361 echo "FATAL: Parameter --install=dev excludes --install=arm-dev" 362 exit 1 363 else 364 INSTALL_DEV=yes 365 fi 366 fi 367 if [[ $FLAG_ARG == "x86" ]] ; then 368 INSTALL_CROSS_x86=yes 369 fi 370 if [[ $FLAG_ARG == "arm-all" ]] ; then 371 INSTALL_CROSS_ARM_ALL=yes 372 INSTALL_CROSS_LIBS=yes 373 fi 374 if [[ $FLAG_ARG == "windows" ]] ; then 375 INSTALL_CROSS_WINDOWS=yes 376 fi 377 if [[ $FLAG_ARG == "ohos-sdk" ]] ; then 378 INSTALL_OHOS_SDK=yes 379 fi 380 if [[ $FLAG_ARG == "llvm-prebuilts" ]] ; then 381 INSTALL_LLVM_PREBUILTS=yes 382 fi 383 if [[ $FLAG_ARG == "arm-dev" ]] ; then 384 if [[ $INSTALL_DEV == "yes" ]] ; then 385 echo "FATAL: Parameter --install=arm-dev excludes --install=dev" 386 exit 1 387 else 388 INSTALL_ARM_DEV=yes 389 fi 390 fi 391 if [[ $FLAG_ARG == "doc-dev" ]] ; then 392 INSTALL_DOC_DEV=yes 393 fi 394 if [[ $FLAG_ARG == "test" ]] ; then 395 INSTALL_TEST=yes 396 fi 397 if [[ $FLAG_ARG == "vmb" ]] ; then 398 INSTALL_VMB=yes 399 fi 400 ;; 401 *) 402 ERROR_ARG="YES" 403 ;; 404 esac 405 406 if [[ -n "${EXTRA_OPTIONS}" ]] 407 then 408 extra_parse "${i}" 409 fi 410 411 if [[ -n "${ERROR_ARG}" ]] 412 then 413 echo "Error: Unsupported flag $i" >&2 414 exit 1 415 fi 416 417done 418 419# 420# Check 'sudo' and if script is running on Ubuntu 421# 422 423if [[ $(id -u) -ne 0 ]] ; then 424 print_help 425 echo "!!!" 426 echo "FATAL: Please run as root." 427 echo "!!!" 428 exit 1 429fi 430 431# Install deps for the script itself. Until this line we assume that 432# only shell builtins and commands from required packages are invoked. 433apt-get update 434install_dep "$SCRIPT_DIR/dep-lists/ubuntu-bootstrap" 435 436# 437# Check specific Ubuntu version 438# 439 440UBUNTU_NAME=ubuntu-20-04 441 442if [ ! -f /etc/os-release ]; then 443 echo "FATAL: /etc/os-release not found. Exiting..." 444 exit 1 445else 446 . /etc/os-release 447 448 if [[ $VERSION_ID == "18.04" ]]; then 449 echo "Installing packages for Ubuntu 18.04 LTS." 450 UBUNTU_NAME=ubuntu-18-04 451 ADD_TOOLCHAIN_REPOS=yes 452 elif [[ $VERSION_ID == "20.04" ]]; then 453 echo "Installing packages for Ubuntu 20.04 LTS." 454 UBUNTU_NAME=ubuntu-20-04 455 ADD_TOOLCHAIN_REPOS=yes 456 elif [[ $VERSION_ID == "22.04" ]]; then 457 echo "Installing packages for Ubuntu 22.04 LTS" 458 UBUNTU_NAME=ubuntu-22-04 459 elif [[ $VERSION_ID == "24.04" ]]; then 460 echo "Installing packages for Ubuntu 24.04 LTS" 461 UBUNTU_NAME=ubuntu-24-04 462 INSTALL_CROSS_LIBS=no 463 else 464 echo "Trying to install packages for Ubuntu with unpinned versions." 465 fi 466 467 if [[ $NAME == "Ubuntu" ]] 468 then 469 set -x 470 apt-get update 471 dpkg -l | grep curl || apt-get -y install curl 472 dpkg -l | grep gnupg || apt-get -y install gnupg 473 474 if [[ -n "${EXTRA_OPTIONS}" ]] 475 then 476 extra_add_repo 477 fi 478 479 set +x 480 else 481 echo "FATAL: Only Ubuntu is supported. This is not. Exiting..." 482 exit 1 483 fi 484fi 485 486set -e 487 488# 489# Install dependencies 490# 491 492if [[ "x$ADD_TOOLCHAIN_REPOS" == "xyes" ]] ; then 493 enable_llvm_repo 14 494 if [[ $VERSION_ID == "18.04" ]]; then 495 enable_test_toolchain_repo 496 fi 497 unpack_clang_rt 14 498fi 499 500install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-build" 501 502if [[ $VERSION_ID == "20.04" ]]; then 503 install_python3_10_ubuntu20_04 504fi 505 506if [[ "x$INSTALL_DEV" == "xyes" ]] ; then 507 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-dev" 508fi 509 510if [[ "x$INSTALL_ARM_DEV" == "xyes" ]] ; then 511 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-arm-dev" 512fi 513 514if [[ "x$INSTALL_CROSS_x86" == "xyes" ]] ; then 515 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-x86" 516fi 517 518if [[ "x$INSTALL_CROSS_WINDOWS" == "xyes" ]] ; then 519 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-windows" 520fi 521 522if [[ "x$INSTALL_OHOS_SDK" == "xyes" ]] ; then 523 "${SCRIPT_DIR}/install-ohos-sdk" 524fi 525 526if [[ "x$INSTALL_LLVM_PREBUILTS" == "xyes" ]] ; then 527 "${SCRIPT_DIR}/install-llvm-prebuilts" 528fi 529 530if [[ "x$INSTALL_CROSS_ARM_ALL" == "xyes" ]] ; then 531 if [[ -z "${EXTRA_OPTIONS}" ]] 532 then 533 "${SCRIPT_DIR}/install-deps-qemu" 534 fi 535 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-arm-all" 536fi 537if [[ "$INSTALL_CROSS_LIBS" == "yes" ]]; then 538 install_cross_libs 539fi 540 541if [[ "x$INSTALL_DOC_DEV" == "xyes" ]]; then 542 if [[ -f "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-doc-dev" ]] ; then 543 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-doc-dev" 544 else 545 install_dep "$SCRIPT_DIR/dep-lists/ubuntu-doc-dev" 546 fi 547 install_pip_dep "${SCRIPT_DIR}/dep-lists/requirements-doc-dev" 548fi 549 550if [[ -n "${EXTRA_OPTIONS}" ]] 551then 552 extra_install 553fi 554 555if [[ "x$INSTALL_TEST" == "xyes" ]]; then 556 if [[ $VERSION_ID != "24.04" ]]; then 557 install_python_dependencies 558 fi 559fi 560 561 562if [[ "x$INSTALL_VMB" == "xyes" ]]; then 563 install_python_dependencies 564 install_pip_dep "${SCRIPT_DIR}/dep-lists/requirements-vmb" 565 build_and_install_vmb 566fi 567