1#!/bin/bash 2# 3# Build and run tests for the protobuf project. We use this script to run 4# tests on kokoro (Ubuntu and MacOS). It can run locally as well but you 5# need to make sure the required compilers/tools are available. 6 7internal_build_cpp() { 8 if [ -f src/protoc ]; then 9 # Already built. 10 return 11 fi 12 13 # Initialize any submodules. 14 git submodule update --init --recursive 15 16 ./autogen.sh 17 ./configure CXXFLAGS="-fPIC -std=c++11" # -fPIC is needed for python cpp test. 18 # See python/setup.py for more details 19 make -j$(nproc) 20} 21 22build_cpp() { 23 internal_build_cpp 24 make check -j$(nproc) || (cat src/test-suite.log; false) 25 cd conformance && make test_cpp && cd .. 26} 27 28build_cpp_tcmalloc() { 29 internal_build_cpp 30 ./configure LIBS=-ltcmalloc && make clean && make \ 31 PTHREAD_CFLAGS='-pthread -DGOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN' \ 32 check 33 cd src 34 PPROF_PATH=/usr/bin/google-pprof HEAPCHECK=strict ./protobuf-test 35} 36 37build_cpp_distcheck() { 38 grep -q -- "-Og" src/Makefile.am && 39 echo "The -Og flag is incompatible with Clang versions older than 4.0." && 40 exit 1 41 42 # Initialize any submodules. 43 git submodule update --init --recursive 44 ./autogen.sh 45 ./configure 46 make dist 47 48 # List all files that should be included in the distribution package. 49 git ls-files | grep "^\(java\|python\|objectivec\|csharp\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ 50 grep -v ".gitignore" | grep -v "java/lite/proguard.pgcfg" |\ 51 grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\ 52 grep -v "python/protobuf_distutils" | grep -v "csharp/compatibility_tests" > dist.lst 53 # Unzip the dist tar file. 54 DIST=`ls *.tar.gz` 55 tar -xf $DIST 56 cd ${DIST//.tar.gz} 57 # Check if every file exists in the dist tar file. 58 FILES_MISSING="" 59 for FILE in $(<../dist.lst); do 60 [ -f "$FILE" ] || { 61 echo "$FILE is not found!" 62 FILES_MISSING="$FILE $FILES_MISSING" 63 } 64 done 65 cd .. 66 if [ ! -z "$FILES_MISSING" ]; then 67 echo "Missing files in EXTRA_DIST: $FILES_MISSING" 68 exit 1 69 fi 70 71 # Do the regular dist-check for C++. 72 make distcheck -j$(nproc) 73} 74 75build_dist_install() { 76 # Create a symlink pointing to python2 and put it at the beginning of $PATH. 77 # This is necessary because the googletest build system involves a Python 78 # script that is not compatible with Python 3. More recent googletest 79 # versions have fixed this, but they have also removed the autotools build 80 # system support that we rely on. This is a temporary workaround to keep the 81 # googletest build working when the default python binary is Python 3. 82 mkdir tmp || true 83 pushd tmp 84 ln -s /usr/bin/python2 ./python 85 popd 86 PATH=$PWD/tmp:$PATH 87 88 # Initialize any submodules. 89 git submodule update --init --recursive 90 ./autogen.sh 91 ./configure 92 make dist 93 94 # Unzip the dist tar file and install it. 95 DIST=`ls *.tar.gz` 96 tar -xf $DIST 97 pushd ${DIST//.tar.gz} 98 ./configure && make check -j4 && make install 99 100 export LD_LIBRARY_PATH=/usr/local/lib 101 102 # Try to install Java 103 pushd java 104 use_java jdk11 105 $MVN install 106 popd 107 108 # Try to install Python 109 python3 -m venv venv 110 source venv/bin/activate 111 pushd python 112 python3 setup.py clean build sdist 113 pip3 install dist/protobuf-*.tar.gz 114 popd 115 deactivate 116 rm -rf python/venv 117} 118 119build_csharp() { 120 # Required for conformance tests and to regenerate protos. 121 internal_build_cpp 122 NUGET=/usr/local/bin/nuget.exe 123 124 # Disable some unwanted dotnet options 125 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true 126 export DOTNET_CLI_TELEMETRY_OPTOUT=true 127 128 # TODO(jtattermusch): is this still needed with "first time experience" 129 # disabled? 130 # Perform "dotnet new" once to get the setup preprocessing out of the 131 # way. That spews a lot of output (including backspaces) into logs 132 # otherwise, and can cause problems. It doesn't matter if this step 133 # is performed multiple times; it's cheap after the first time anyway. 134 # (It also doesn't matter if it's unnecessary, which it will be on some 135 # systems. It's necessary on Jenkins in order to avoid unprintable 136 # characters appearing in the JUnit output.) 137 mkdir dotnettmp 138 (cd dotnettmp; dotnet new > /dev/null) 139 rm -rf dotnettmp 140 141 # Check that the protos haven't broken C# codegen. 142 # TODO(jonskeet): Fail if regenerating creates any changes. 143 csharp/generate_protos.sh 144 145 csharp/buildall.sh 146 cd conformance && make test_csharp && cd .. 147 148 # Run csharp compatibility test between 3.0.0 and the current version. 149 csharp/compatibility_tests/v3.0.0/test.sh 3.0.0 150 151 # Regression test for https://github.com/protocolbuffers/protobuf/issues/9526 152 # - all line endings in .proto and .cs (and .csproj) files should be LF. 153 if git ls-files --eol csharp | grep -E '\.cs|\.proto' | grep -v w/lf 154 then 155 echo "The files listed above have mixed or CRLF line endings; please change to LF." 156 exit 1 157 fi 158} 159 160build_golang() { 161 # Go build needs `protoc`. 162 internal_build_cpp 163 # Add protoc to the path so that the examples build finds it. 164 export PATH="`pwd`/src:$PATH" 165 166 export GOPATH="$HOME/gocode" 167 mkdir -p "$GOPATH/src/github.com/protocolbuffers" 168 mkdir -p "$GOPATH/src/github.com/golang" 169 rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf" 170 rm -f "$GOPATH/src/github.com/golang/protobuf" 171 ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf" 172 export PATH="$GOPATH/bin:$PATH" 173 (cd $GOPATH/src/github.com/golang && git clone https://github.com/golang/protobuf.git && cd protobuf && git checkout v1.3.5) 174 go install github.com/golang/protobuf/protoc-gen-go 175 176 cd examples && PROTO_PATH="-I../src -I." make gotest && cd .. 177} 178 179use_java() { 180 version=$1 181 case "$version" in 182 jdk17) 183 export PATH=/usr/lib/jvm/java-17-openjdk-amd64/bin:$PATH 184 export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 185 ;; 186 jdk11) 187 export PATH=/usr/lib/jvm/java-11-openjdk-amd64/bin:$PATH 188 export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 189 ;; 190 jdk8) 191 export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH 192 export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 193 ;; 194 jdk7) 195 export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH 196 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 197 ;; 198 oracle7) 199 export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH 200 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 201 ;; 202 esac 203 204 MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository 205 MVN="$MVN -e --quiet -Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" 206 207 which java 208 java -version 209 $MVN -version 210} 211 212# --batch-mode suppresses download progress output that spams the logs. 213MVN="mvn --batch-mode" 214 215internal_build_java() { 216 version=$1 217 dir=java_$version 218 # Java build needs `protoc`. 219 internal_build_cpp 220 cp -r java $dir 221 cd $dir && $MVN clean 222 # Skip tests here - callers will decide what tests they want to run 223 $MVN install -Dmaven.test.skip=true 224} 225 226build_java() { 227 version=$1 228 internal_build_java $version 229 # Skip the Kotlin tests on Oracle 7 230 if [ "$version" == "oracle7" ]; then 231 $MVN test -pl bom,lite,core,util 232 else 233 $MVN test 234 fi 235 cd ../.. 236} 237 238# The conformance tests are hard-coded to work with the $ROOT/java directory. 239# So this can't run in parallel with two different sets of tests. 240build_java_with_conformance_tests() { 241 # Java build needs `protoc`. 242 internal_build_cpp 243 # This local installation avoids the problem caused by a new version not yet in Maven Central 244 cd java/bom && $MVN install 245 cd ../.. 246 cd java/core && $MVN test && $MVN install 247 cd ../lite && $MVN test && $MVN install 248 cd ../util && $MVN test && $MVN install && $MVN package assembly:single 249 if [ "$version" == "jdk8" ]; then 250 cd ../kotlin && $MVN test && $MVN install 251 cd ../kotlin-lite && $MVN test && $MVN install 252 fi 253 cd ../.. 254 cd conformance && make test_java && cd .. 255} 256 257build_java_jdk7() { 258 use_java jdk7 259 build_java_with_conformance_tests 260} 261 262build_java_oracle7() { 263 use_java oracle7 264 build_java oracle7 265} 266 267build_java_jdk8() { 268 use_java jdk8 269 build_java_with_conformance_tests 270} 271 272build_java_jdk11() { 273 use_java jdk11 274 build_java 275} 276 277build_java_jdk17() { 278 use_java jdk17 279 build_java 280} 281 282build_java_linkage_monitor() { 283 # Linkage Monitor checks compatibility with other Google libraries 284 # https://github.com/GoogleCloudPlatform/cloud-opensource-java/tree/master/linkage-monitor 285 286 use_java jdk11 287 internal_build_cpp 288 289 # Linkage Monitor uses $HOME/.m2 local repository 290 MVN="mvn -e -B -Dhttps.protocols=TLSv1.2" 291 cd java 292 # Installs the snapshot version locally 293 $MVN install -Dmaven.test.skip=true 294 295 # Linkage Monitor uses the snapshot versions installed in $HOME/.m2 to verify compatibility 296 JAR=linkage-monitor-latest-all-deps.jar 297 curl -v -O "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/${JAR}" 298 # Fails if there's new linkage errors compared with baseline 299 java -jar $JAR com.google.cloud:libraries-bom 300} 301 302build_objectivec_ios() { 303 # Reused the build script that takes care of configuring and ensuring things 304 # are up to date. The OS X test runs the objc conformance test, so skip it 305 # here. 306 objectivec/DevTools/full_mac_build.sh \ 307 --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance "$@" 308} 309 310build_objectivec_ios_debug() { 311 build_objectivec_ios --skip-xcode-release 312} 313 314build_objectivec_ios_release() { 315 build_objectivec_ios --skip-xcode-debug 316} 317 318build_objectivec_osx() { 319 # Reused the build script that takes care of configuring and ensuring things 320 # are up to date. 321 objectivec/DevTools/full_mac_build.sh \ 322 --core-only --skip-xcode-ios --skip-xcode-tvos 323} 324 325build_objectivec_tvos() { 326 # Reused the build script that takes care of configuring and ensuring things 327 # are up to date. The OS X test runs the objc conformance test, so skip it 328 # here. 329 objectivec/DevTools/full_mac_build.sh \ 330 --core-only --skip-xcode-ios --skip-xcode-osx --skip-objc-conformance "$@" 331} 332 333build_objectivec_tvos_debug() { 334 build_objectivec_tvos --skip-xcode-release 335} 336 337build_objectivec_tvos_release() { 338 build_objectivec_tvos --skip-xcode-debug 339} 340 341build_python() { 342 internal_build_cpp 343 cd python 344 tox --skip-missing-interpreters 345 cd .. 346} 347 348build_python_version() { 349 internal_build_cpp 350 cd python 351 envlist=$1 352 tox -e $envlist 353 cd .. 354} 355 356build_python37() { 357 build_python_version py37-python 358} 359 360build_python38() { 361 build_python_version py38-python 362} 363 364build_python39() { 365 build_python_version py39-python 366} 367 368build_python310() { 369 build_python_version py310-python 370} 371 372build_python_cpp() { 373 internal_build_cpp 374 export LD_LIBRARY_PATH=../src/.libs # for Linux 375 export DYLD_LIBRARY_PATH=../src/.libs # for OS X 376 cd python 377 tox --skip-missing-interpreters 378 cd .. 379} 380 381build_python_cpp_version() { 382 internal_build_cpp 383 export LD_LIBRARY_PATH=../src/.libs # for Linux 384 export DYLD_LIBRARY_PATH=../src/.libs # for OS X 385 cd python 386 envlist=$1 387 tox -e $envlist 388 cd .. 389} 390 391build_python37_cpp() { 392 build_python_cpp_version py37-cpp 393} 394 395build_python38_cpp() { 396 build_python_cpp_version py38-cpp 397} 398 399build_python39_cpp() { 400 build_python_cpp_version py39-cpp 401} 402 403build_python310_cpp() { 404 build_python_cpp_version py310-cpp 405} 406 407build_ruby23() { 408 internal_build_cpp # For conformance tests. 409 cd ruby && bash travis-test.sh ruby-2.3.8 && cd .. 410} 411build_ruby24() { 412 internal_build_cpp # For conformance tests. 413 cd ruby && bash travis-test.sh ruby-2.4 && cd .. 414} 415build_ruby25() { 416 internal_build_cpp # For conformance tests. 417 cd ruby && bash travis-test.sh ruby-2.5.1 && cd .. 418} 419build_ruby26() { 420 internal_build_cpp # For conformance tests. 421 cd ruby && bash travis-test.sh ruby-2.6.0 && cd .. 422} 423build_ruby27() { 424 internal_build_cpp # For conformance tests. 425 cd ruby && bash travis-test.sh ruby-2.7.0 && cd .. 426} 427build_ruby30() { 428 internal_build_cpp # For conformance tests. 429 cd ruby && bash travis-test.sh ruby-3.0.2 && cd .. 430} 431build_ruby31() { 432 internal_build_cpp # For conformance tests. 433 cd ruby && bash travis-test.sh ruby-3.1.0 && cd .. 434} 435 436build_jruby92() { 437 internal_build_cpp # For conformance tests. 438 internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes 439 cd ruby && bash travis-test.sh jruby-9.2.20.1 && cd .. 440} 441 442build_jruby93() { 443 internal_build_cpp # For conformance tests. 444 internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes 445 cd ruby && bash travis-test.sh jruby-9.3.4.0 && cd .. 446} 447 448use_php() { 449 VERSION=$1 450 export PATH=/usr/local/php-${VERSION}/bin:$PATH 451 internal_build_cpp 452} 453 454build_php() { 455 use_php $1 456 pushd php 457 rm -rf vendor 458 php -v 459 php -m 460 composer update 461 composer test 462 popd 463 (cd conformance && make test_php) 464} 465 466test_php_c() { 467 pushd php 468 rm -rf vendor 469 php -v 470 php -m 471 composer update 472 composer test_c 473 popd 474 (cd conformance && make test_php_c) 475} 476 477build_php_c() { 478 use_php $1 479 test_php_c 480} 481 482build_php7.0_mac() { 483 internal_build_cpp 484 # Install PHP 485 curl -s https://php-osx.liip.ch/install.sh | bash -s 7.0 486 PHP_FOLDER=`find /usr/local -type d -name "php5-7.0*"` # The folder name may change upon time 487 test ! -z "$PHP_FOLDER" 488 export PATH="$PHP_FOLDER/bin:$PATH" 489 490 # Install Composer 491 wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer 492 chmod a+x /usr/local/bin/composer 493 494 # Install valgrind 495 echo "#! /bin/bash" > valgrind 496 chmod ug+x valgrind 497 sudo mv valgrind /usr/local/bin/valgrind 498 499 # Test 500 test_php_c 501} 502 503build_php7.3_mac() { 504 internal_build_cpp 505 # Install PHP 506 # We can't test PHP 7.4 with these binaries yet: 507 # https://github.com/liip/php-osx/issues/276 508 curl -s https://php-osx.liip.ch/install.sh | bash -s 7.3 509 PHP_FOLDER=`find /usr/local -type d -name "php5-7.3*"` # The folder name may change upon time 510 test ! -z "$PHP_FOLDER" 511 export PATH="$PHP_FOLDER/bin:$PATH" 512 513 # Install Composer 514 wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer 515 chmod a+x /usr/local/bin/composer 516 517 # Install valgrind 518 echo "#! /bin/bash" > valgrind 519 chmod ug+x valgrind 520 sudo mv valgrind /usr/local/bin/valgrind 521 522 # Test 523 test_php_c 524} 525 526build_php_compatibility() { 527 internal_build_cpp 528} 529 530build_php_multirequest() { 531 use_php 7.4 532 php/tests/multirequest.sh 533} 534 535build_php8.0_all() { 536 build_php 8.0 537 build_php 8.1 538 build_php_c 8.0 539 build_php_c 8.1 540} 541 542build_php_all_32() { 543 build_php 7.0 544 build_php 7.1 545 build_php 7.4 546 build_php_c 7.0 547 build_php_c 7.1 548 build_php_c 7.4 549 build_php_c 7.1-zts 550 build_php_c 7.2-zts 551 build_php_c 7.5-zts 552} 553 554build_php_all() { 555 build_php_all_32 556 build_php_multirequest 557 build_php_compatibility 558} 559 560build_benchmark() { 561 use_php 7.2 562 cd kokoro/linux/benchmark && ./run.sh 563} 564 565# -------- main -------- 566 567if [ "$#" -ne 1 ]; then 568 echo " 569Usage: $0 { cpp | 570 cpp_distcheck | 571 csharp | 572 java_jdk7 | 573 java_oracle7 | 574 java_jdk8 | 575 java_jdk11 | 576 java_jdk17 | 577 java_linkage_monitor | 578 objectivec_ios | 579 objectivec_ios_debug | 580 objectivec_ios_release | 581 objectivec_osx | 582 objectivec_tvos | 583 objectivec_tvos_debug | 584 objectivec_tvos_release | 585 python | 586 python_cpp | 587 python_compatibility | 588 ruby23 | 589 ruby24 | 590 ruby25 | 591 ruby26 | 592 ruby27 | 593 ruby30 | 594 ruby31 | 595 jruby92 | 596 jruby93 | 597 ruby_all | 598 php_all | 599 php_all_32 | 600 php7.0_mac | 601 php7.3_mac | 602 dist_install | 603 benchmark } 604" 605 exit 1 606fi 607 608set -e # exit immediately on error 609set -x # display all commands 610cd $(dirname $0) 611eval "build_$1" 612