1#!/usr/bin/env bash 2# Copyright (c) 2021-2024 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# 16# Aux functions 17# 18 19function enable_llvm_repo 20{ 21 set -x 22 local llvm_url=${LLVM_REPO:-'https://apt.llvm.org'} 23 local llvm_version=$1 24 local llvm_gpg_url=${LLVM_GPG_REPO:-'https://apt.llvm.org'} 25 26 local repo_name="deb ${llvm_url}/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${llvm_version} main" 27 curl --retry 5 --retry-delay 10 -k ${llvm_gpg_url}/llvm-snapshot.gpg.key | apt-key add - 28 echo -e $repo_name > /etc/apt/sources.list.d/llvm-${llvm_version}.list 29 apt-get update 30 set +x 31} 32 33function unpack_clang_rt 34{ 35 local llvm_version=$1 36 37 # install requirements 38 apt-get update 39 apt-get install -y --no-install-recommends \ 40 binutils \ 41 gzip \ 42 tar \ 43 xz-utils 44 mkdir -p clang_rt && cd clang_rt 45 apt-get download libclang-rt-${llvm_version}-dev 46 ar x libclang-rt-${llvm_version}-dev*.deb 47 tar xJf data.tar.xz 48 cp -r usr / 49 cd ../ 50 rm -rf clang_rt 51} 52 53SCRIPT_DIR="$(realpath "${0}")" 54SCRIPT_DIR="$(dirname "${SCRIPT_DIR}")" 55cd "${SCRIPT_DIR}" 56 57if [[ -f "${SCRIPT_DIR}/extras/install-deps-extras.sh" ]] 58then 59 source "${SCRIPT_DIR}/extras/install-deps-extras.sh" 60fi 61 62function print_help 63{ 64 HELP_MESSAGE=" 65 It is the bootstrap script for Panda on Ubuntu 18.04 or 20.04. 66 67 This script installs all necessary packages for building and testing Panda 68 in your local environment, given that your environment is Ubuntu 18.04 or 20.04. 69 (detected with the contents of /etc/os-release). 70 71 The script should run with superuser privileges. 72 73 EXAMPLE 74 75 $ ./scripts/install-deps-ubuntu --help 76 $ ./scripts/install-deps-ubuntu --install=x86 --install=arm-all --install=dev 77 78 or 79 80 $ ./scripts/install-deps-ubuntu -h 81 $ ./scripts/install-deps-ubuntu -i=x86 -i=arm-all -i=dev 82 83 SYNOPSIS 84 85 $0 [OPTIONS] 86 87 OPTIONS 88 89 --help | -h Show this message and exit 90 91 --install=dev | -i=dev Install tools needed for development 92 93 --install=arm-dev | -i=arm-dev Install ARM64-hosted tools needed for development 94 95 --install=arm-all | -i=arm-all Install extra packages for cross-compiling for AArch32 and AArch64 96 97 --install=x86 | -i=x86 Install extra packages for cross-compiling for x86 98 99 --install=windows | -i=windows Install extra packages for cross-compiling for Windows 100 101 --install=doc-dev | -i=doc-dev Install tools for documentation development 102 103 --install=test | -i=test Install python dependencies for tests running 104 " 105 106 if [[ -n "${EXTRA_OPTIONS}" ]] 107 then 108 HELP_MESSAGE="${HELP_MESSAGE}${ADDITIONAL_OPTIONS_HELP}" 109 fi 110 111 112 HELP_MESSAGE="${HELP_MESSAGE} 113 CAVEAT 114 115 * Packages for cross-compiling for aarch64 and x86 cannot co-exist, so the 116 script (read: apt) will replace any conflicting packages on each run. 117 * However, packages for cross-compiling for aarch64 and 32-bit ARM can 118 co-exist, so they are in a single 'arm-all' dependency list. 119 " 120 121 echo "$HELP_MESSAGE" 122} 123 124function install_dep 125{ 126 local fname=$1 127 128 if [[ ! -f "$fname" ]] ; then 129 echo "FATAL: Dependency list $fname not found." 130 exit 1 131 fi 132 133 echo "Processing $fname" 134 grep --color=never -o '^[^#]*' "$fname" | xargs apt install -y --no-install-recommends -o Dpkg::Options::="--force-overwrite" -o Acquire::Retries=30 -o Acquire::https::Timeout=600 -o Acquire::http::Timeout=600 135} 136 137function install_pip_dep 138{ 139 local fname=$1 140 141 if [[ ! -f "$fname" ]] ; then 142 echo "FATAL: Dependency list $fname not found." 143 exit 1 144 fi 145 146 echo "Processing $fname" 147 python3 -m pip install --no-cache-dir --upgrade -r $fname 148} 149 150function enable_test_toolchain_repo 151{ 152 set -x 153 local repo_url=${PPA_REPO:-'https://ppa.launchpadcontent.net/ubuntu-toolchain-r/test/ubuntu'} 154 155 local repo_name="deb ${repo_url} ${UBUNTU_CODENAME} main" 156 curl --retry 5 --retry-delay 10 -k "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x60C317803A41BA51845E371A1E9377A2BA9EF27F" | apt-key add - 157 echo -e $repo_name > /etc/apt/sources.list.d/ubuntu-toolchain-r-test.list 158 159 cat > /etc/apt/preferences.d/ubuntu-toolchain-r-test << EOF 160Package: * 161Pin: release o=LP-PPA-ubuntu-toolchain-r-test 162Pin-Priority: 1 163EOF 164 165 apt-get update 166 set +x 167} 168 169function install_cross_zlib 170{ 171 local zlib_url=${ZLIB_URL:-'https://zlib.net/fossils/zlib-1.2.11.tar.gz'} 172 TEMP=`mktemp -d` 173 pushd . 174 cd $TEMP 175 curl --retry 5 --retry-delay 10 -o zlib.tar.gz "${zlib_url}" 176 tar xf zlib.tar.gz 177 cd zlib-1.2.11 178 179 if [ `which aarch64-linux-gnu-gcc-8` ]; then 180 CC=aarch64-linux-gnu-gcc-8 ./configure --prefix=/usr/aarch64-linux-gnu 181 make -j install 182 make clean 183 fi 184 185 if [ `which arm-linux-gnueabi-gcc-8` ]; then 186 CC=arm-linux-gnueabi-gcc-8 ./configure --prefix=/usr/arm-linux-gnueabi 187 make -j install 188 make clean 189 fi 190 191 if [ `which arm-linux-gnueabihf-gcc-8` ]; then 192 CC=arm-linux-gnueabihf-gcc-8 ./configure --prefix=/usr/arm-linux-gnueabihf 193 make -j install 194 make clean 195 fi 196 197 popd 198 rm -rf $TEMP 199} 200 201function install_cross_libdwarf 202{ 203 local lib_url=${LIBDWRF_URL:-'https://codeload.github.com/davea42/libdwarf-code/tar.gz/refs/tags/20201020'} 204 TEMP=`mktemp -d` 205 pushd . 206 cd $TEMP 207 curl --retry 5 --retry-delay 10 -o libdwarf.tar.gz "${lib_url}" 208 tar xf libdwarf.tar.gz 209 cd libdwarf-code-20201020 210 autoreconf 211 212 if [ `which aarch64-linux-gnu-gcc-8` ]; then 213 CC=aarch64-linux-gnu-gcc-8 ./configure --host=aarch64-linux-gnu --prefix=/usr/aarch64-linux-gnu --includedir=/usr/aarch64-linux-gnu/include/libdwarf --disable-libelf --enable-shared --with-pic 214 make -j install 215 make clean 216 fi 217 218 if [ `which arm-linux-gnueabi-gcc-8` ]; then 219 CC=arm-linux-gnueabi-gcc-8 ./configure --host=arm-linux-gnueabi --prefix=/usr/arm-linux-gnueabi --includedir=/usr/arm-linux-gnueabi/include/libdwarf --disable-libelf --enable-shared --with-pic 220 make -j install 221 make clean 222 fi 223 224 if [ `which arm-linux-gnueabihf-gcc-8` ]; then 225 CC=arm-linux-gnueabihf-gcc-8 ./configure --host=arm-linux-gnueabihf --prefix=/usr/arm-linux-gnueabihf --includedir=/usr/arm-linux-gnueabihf/include/libdwarf --disable-libelf --enable-shared --with-pic 226 make -j install 227 make clean 228 fi 229 230 popd 231 rm -rf $TEMP 232} 233 234function install_cross_libs 235{ 236 echo "Installing libraries for cross-compilation ..." 237 apt install -y automake 238 install_cross_zlib 239 install_cross_libdwarf 240 echo "Success" 241} 242 243function install_python_dependencies 244{ 245 if [[ $VERSION_ID == "18.04" ]]; then 246 echo "Install python 3.8" 247 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 248 fi 249 250 local REQUIREMENTS_BASE_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-base-python3 251 local REQUIREMENTS_ALL_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-python3 252 local REQUIREMENTS_VENV_PYTHON3_NAME=$SCRIPT_DIR/dep-lists/requirements-venv-python3 253 254 echo "Installing pip packages for all" 255 install_pip_dep "${REQUIREMENTS_BASE_PYTHON3_NAME}" 256 install_pip_dep "${REQUIREMENTS_ALL_PYTHON3_NAME}" 257 258 echo "Creating venv and installing pip packages there" 259 local MY_USERNAME=${SUDO_USER} 260 if [[ -z "${VENV_DIR}" && -n "${MY_USERNAME}" ]]; then 261 local MY_HOME=$(grep "^${MY_USERNAME}:" /etc/passwd | cut -d: -f6) 262 if [[ ! -e "${MY_HOME}" ]]; then 263 MY_HOME=/home/${MY_USERNAME} 264 fi 265 VENV_DIR=${MY_HOME}/.venv-panda 266 elif [[ -z "${VENV_DIR}" && -z "${MY_USERNAME}" ]]; then 267 VENV_DIR=/root/.venv-panda 268 fi 269 270 echo "Going to create a virtual environment at ${VENV_DIR}" 271 python3 -m venv "${VENV_DIR}" 272 273 source "${VENV_DIR}/bin/activate" 274 install_pip_dep "${REQUIREMENTS_BASE_PYTHON3_NAME}" 275 install_pip_dep "${REQUIREMENTS_VENV_PYTHON3_NAME}" 276 deactivate 277 278 if [[ -n "${MY_USERNAME}" ]]; then 279 local MY_UID=$(id -u "${MY_USERNAME}") 280 local MY_GID=$(id -g "${MY_USERNAME}") 281 chown -R $MY_UID:$MY_GID $VENV_DIR 282 fi 283} 284 285# 286# Main logic 287# 288 289# 290# Parse command-line arguments 291# 292 293# Set default flag values 294INSTALL_DEV=no 295INSTALL_CROSS_x86=no 296INSTALL_CROSS_WINDOWS=no 297INSTALL_ARM_DEV=no 298INSTALL_CROSS_ARM_ALL=no 299INSTALL_CROSS_LIBS=no 300INSTALL_DOC_DEV=no 301INSTALL_TEST=no 302ADD_TOOLCHAIN_REPOS=no 303SRC_LIST_STR='# This file is generated automatically by Panda install-deps-ubuntu script. DO NOT EDIT!!!\n' 304 305for i in "$@" 306do 307 ERROR_ARG="" 308 case $i in 309 -h|--help) 310 print_help 311 exit 0 312 ;; 313 -i=*|--install=*) 314 FLAG_ARG=${i//[-a-zA-Z0-9]*=/} 315 if [[ $FLAG_ARG == "dev" ]] ; then 316 if [[ $INSTALL_ARM_DEV == "yes" ]] ; then 317 echo "FATAL: Parameter --install=dev excludes --install=arm-dev" 318 exit 1 319 else 320 INSTALL_DEV=yes 321 fi 322 fi 323 if [[ $FLAG_ARG == "x86" ]] ; then 324 INSTALL_CROSS_x86=yes 325 fi 326 if [[ $FLAG_ARG == "arm-all" ]] ; then 327 INSTALL_CROSS_ARM_ALL=yes 328 INSTALL_CROSS_LIBS=yes 329 fi 330 if [[ $FLAG_ARG == "windows" ]] ; then 331 INSTALL_CROSS_WINDOWS=yes 332 fi 333 if [[ $FLAG_ARG == "arm-dev" ]] ; then 334 if [[ $INSTALL_DEV == "yes" ]] ; then 335 echo "FATAL: Parameter --install=arm-dev excludes --install=dev" 336 exit 1 337 else 338 INSTALL_ARM_DEV=yes 339 fi 340 fi 341 if [[ $FLAG_ARG == "doc-dev" ]] ; then 342 INSTALL_DOC_DEV=yes 343 fi 344 if [[ $FLAG_ARG == "test" ]] ; then 345 INSTALL_TEST=yes 346 fi 347 ;; 348 *) 349 ERROR_ARG="YES" 350 ;; 351 esac 352 353 if [[ -n "${EXTRA_OPTIONS}" ]] 354 then 355 extra_parse "${i}" 356 fi 357 358 if [[ -n "${ERROR_ARG}" ]] 359 then 360 echo "Error: Unsupported flag $i" >&2 361 exit 1 362 fi 363 364done 365 366# 367# Check 'sudo' and if script is running on Ubuntu 368# 369 370if [[ $(id -u) -ne 0 ]] ; then 371 print_help 372 echo "!!!" 373 echo "FATAL: Please run as root." 374 echo "!!!" 375 exit 1 376fi 377 378# Install deps for the script itself. Until this line we assume that 379# only shell builtins and commands from required packages are invoked. 380apt-get update 381install_dep "$SCRIPT_DIR/dep-lists/ubuntu-bootstrap" 382 383# 384# Check specific Ubuntu version 385# 386 387UBUNTU_NAME=ubuntu-18-04 388 389if [ ! -f /etc/os-release ]; then 390 echo "FATAL: /etc/os-release not found. Exiting..." 391 exit 1 392else 393 . /etc/os-release 394 395 if [[ $VERSION_ID == "18.04" ]]; then 396 echo "Installing packages for Ubuntu 18.04 LTS." 397 UBUNTU_NAME=ubuntu-18-04 398 ADD_TOOLCHAIN_REPOS=yes 399 elif [[ $VERSION_ID == "20.04" ]]; then 400 echo "Installing packages for Ubuntu 20.04 LTS." 401 UBUNTU_NAME=ubuntu-20-04 402 ADD_TOOLCHAIN_REPOS=yes 403 elif [[ $VERSION_ID == "22.04" ]]; then 404 echo "Installing packages for Ubuntu 22.04 LTS" 405 UBUNTU_NAME=ubuntu-22-04 406 else 407 echo "Trying to install packages for Ubuntu with unpinned versions." 408 fi 409 410 if [[ $NAME == "Ubuntu" ]] 411 then 412 set -x 413 apt-get update 414 dpkg -l | grep curl || apt-get -y install curl 415 dpkg -l | grep gnupg || apt-get -y install gnupg 416 417 if [[ -n "${EXTRA_OPTIONS}" ]] 418 then 419 extra_add_repo 420 fi 421 422 set +x 423 else 424 echo "FATAL: Only Ubuntu is supported. This is not. Exiting..." 425 exit 1 426 fi 427fi 428 429set -e 430 431# 432# Install dependencies 433# 434 435if [[ "x$ADD_TOOLCHAIN_REPOS" == "xyes" ]] ; then 436 enable_llvm_repo 14 437 if [[ $VERSION_ID == "18.04" ]]; then 438 enable_test_toolchain_repo 439 fi 440 unpack_clang_rt 14 441fi 442 443install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-build" 444 445if [[ "x$INSTALL_DEV" == "xyes" ]] ; then 446 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-dev" 447fi 448 449if [[ "x$INSTALL_ARM_DEV" == "xyes" ]] ; then 450 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-arm-dev" 451fi 452 453if [[ "x$INSTALL_CROSS_x86" == "xyes" ]] ; then 454 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-x86" 455fi 456 457if [[ "x$INSTALL_CROSS_WINDOWS" == "xyes" ]] ; then 458 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-windows" 459fi 460 461if [[ "x$INSTALL_CROSS_ARM_ALL" == "xyes" ]] ; then 462 if [[ -z "${EXTRA_OPTIONS}" ]] 463 then 464 "${SCRIPT_DIR}/install-deps-qemu" 465 fi 466 install_dep "$SCRIPT_DIR/dep-lists/$UBUNTU_NAME-cross-arm-all" 467fi 468if [[ "$INSTALL_CROSS_LIBS" == "yes" ]]; then 469 install_cross_libs 470fi 471 472if [[ "x$INSTALL_DOC_DEV" == "xyes" ]]; then 473 install_dep "$SCRIPT_DIR/dep-lists/ubuntu-doc-dev" 474 install_pip_dep "${SCRIPT_DIR}/dep-lists/requirements-doc-dev" 475fi 476 477if [[ -n "${EXTRA_OPTIONS}" ]] 478then 479 extra_install 480fi 481 482if [[ "x$INSTALL_TEST" == "xyes" ]]; then 483 install_python_dependencies 484fi 485