1#!/bin/bash 2# 3# Build and runs tests for the protobuf project. We use this script to run 4# tests on kokoro (Ubuntu and MacOS). It can run locally as well but you 5# will need to make sure the required compilers/tools are available. 6 7# For when some other test needs the C++ main build, including protoc and 8# libprotobuf. 9LAST_RELEASED=3.9.0 10 11internal_build_cpp() { 12 if [ -f src/protoc ]; then 13 # Already built. 14 return 15 fi 16 17 # Initialize any submodules. 18 git submodule update --init --recursive 19 20 ./autogen.sh 21 ./configure CXXFLAGS="-fPIC -std=c++11" # -fPIC is needed for python cpp test. 22 # See python/setup.py for more details 23 make -j$(nproc) 24} 25 26build_cpp() { 27 internal_build_cpp 28 make check -j$(nproc) || (cat src/test-suite.log; false) 29 cd conformance && make test_cpp && cd .. 30 31 # The benchmark code depends on cmake, so test if it is installed before 32 # trying to do the build. 33 if [[ $(type cmake 2>/dev/null) ]]; then 34 # Verify benchmarking code can build successfully. 35 cd benchmarks && make cpp-benchmark && cd .. 36 else 37 echo "" 38 echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed." 39 echo "" 40 fi 41} 42 43build_cpp_tcmalloc() { 44 internal_build_cpp 45 ./configure LIBS=-ltcmalloc && make clean && make \ 46 PTHREAD_CFLAGS='-pthread -DGOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN' \ 47 check 48 cd src 49 PPROF_PATH=/usr/bin/google-pprof HEAPCHECK=strict ./protobuf-test 50} 51 52build_cpp_distcheck() { 53 grep -q -- "-Og" src/Makefile.am && 54 echo "The -Og flag is incompatible with Clang versions older than 4.0." && 55 exit 1 56 57 # Initialize any submodules. 58 git submodule update --init --recursive 59 ./autogen.sh 60 ./configure 61 make dist 62 63 # List all files that should be included in the distribution package. 64 git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ 65 grep -v ".gitignore" | grep -v "java/compatibility_tests" | grep -v "java/lite/proguard.pgcfg" |\ 66 grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\ 67 grep -v "csharp/compatibility_tests" > dist.lst 68 # Unzip the dist tar file. 69 DIST=`ls *.tar.gz` 70 tar -xf $DIST 71 cd ${DIST//.tar.gz} 72 # Check if every file exists in the dist tar file. 73 FILES_MISSING="" 74 for FILE in $(<../dist.lst); do 75 [ -f "$FILE" ] || { 76 echo "$FILE is not found!" 77 FILES_MISSING="$FILE $FILES_MISSING" 78 } 79 done 80 cd .. 81 if [ ! -z "$FILES_MISSING" ]; then 82 echo "Missing files in EXTRA_DIST: $FILES_MISSING" 83 exit 1 84 fi 85 86 # Do the regular dist-check for C++. 87 make distcheck -j$(nproc) 88} 89 90build_dist_install() { 91 # Initialize any submodules. 92 git submodule update --init --recursive 93 ./autogen.sh 94 ./configure 95 make dist 96 97 # Unzip the dist tar file and install it. 98 DIST=`ls *.tar.gz` 99 tar -xf $DIST 100 pushd ${DIST//.tar.gz} 101 ./configure && make check -j4 && make install 102 103 export LD_LIBRARY_PATH=/usr/local/lib 104 105 # Try to install Java 106 pushd java 107 use_java jdk7 108 $MVN install 109 popd 110 111 # Try to install Python 112 virtualenv --no-site-packages venv 113 source venv/bin/activate 114 pushd python 115 python setup.py clean build sdist 116 pip install dist/protobuf-*.tar.gz 117 popd 118 deactivate 119 rm -rf python/venv 120} 121 122build_csharp() { 123 # Required for conformance tests and to regenerate protos. 124 internal_build_cpp 125 NUGET=/usr/local/bin/nuget.exe 126 127 # Disable some unwanted dotnet options 128 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true 129 export DOTNET_CLI_TELEMETRY_OPTOUT=true 130 131 # TODO(jtattermusch): is this still needed with "first time experience" 132 # disabled? 133 # Perform "dotnet new" once to get the setup preprocessing out of the 134 # way. That spews a lot of output (including backspaces) into logs 135 # otherwise, and can cause problems. It doesn't matter if this step 136 # is performed multiple times; it's cheap after the first time anyway. 137 # (It also doesn't matter if it's unnecessary, which it will be on some 138 # systems. It's necessary on Jenkins in order to avoid unprintable 139 # characters appearing in the JUnit output.) 140 mkdir dotnettmp 141 (cd dotnettmp; dotnet new > /dev/null) 142 rm -rf dotnettmp 143 144 # Check that the protos haven't broken C# codegen. 145 # TODO(jonskeet): Fail if regenerating creates any changes. 146 csharp/generate_protos.sh 147 148 csharp/buildall.sh 149 cd conformance && make test_csharp && cd .. 150 151 # Run csharp compatibility test between 3.0.0 and the current version. 152 csharp/compatibility_tests/v3.0.0/test.sh 3.0.0 153 154 # Run csharp compatibility test between last released and the current version. 155 csharp/compatibility_tests/v3.0.0/test.sh $LAST_RELEASED 156} 157 158build_golang() { 159 # Go build needs `protoc`. 160 internal_build_cpp 161 # Add protoc to the path so that the examples build finds it. 162 export PATH="`pwd`/src:$PATH" 163 164 export GOPATH="$HOME/gocode" 165 mkdir -p "$GOPATH/src/github.com/protocolbuffers" 166 mkdir -p "$GOPATH/src/github.com/golang" 167 rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf" 168 rm -f "$GOPATH/src/github.com/golang/protobuf" 169 ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf" 170 export PATH="$GOPATH/bin:$PATH" 171 (cd $GOPATH/src/github.com/golang && git clone https://github.com/golang/protobuf.git && cd protobuf && git checkout v1.3.5) 172 go install github.com/golang/protobuf/protoc-gen-go 173 174 cd examples && PROTO_PATH="-I../src -I." make gotest && cd .. 175} 176 177use_java() { 178 version=$1 179 case "$version" in 180 jdk8) 181 export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH 182 export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 183 ;; 184 jdk7) 185 export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH 186 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 187 ;; 188 oracle7) 189 export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH 190 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 191 ;; 192 esac 193 194 MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository 195 MVN="$MVN -e -X -Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" 196 197 which java 198 java -version 199 $MVN -version 200} 201 202# --batch-mode suppresses download progress output that spams the logs. 203MVN="mvn --batch-mode" 204 205build_java() { 206 version=$1 207 dir=java_$version 208 # Java build needs `protoc`. 209 internal_build_cpp 210 cp -r java $dir 211 cd $dir && $MVN clean && $MVN test 212 cd ../.. 213} 214 215# The conformance tests are hard-coded to work with the $ROOT/java directory. 216# So this can't run in parallel with two different sets of tests. 217build_java_with_conformance_tests() { 218 # Java build needs `protoc`. 219 internal_build_cpp 220 # This local installation avoids the problem caused by a new version not yet in Maven Central 221 cd java/bom && $MVN install 222 cd ../.. 223 cd java && $MVN test && $MVN install 224 cd util && $MVN package assembly:single 225 cd ../.. 226 cd conformance && make test_java && cd .. 227} 228 229build_java_jdk7() { 230 use_java jdk7 231 build_java_with_conformance_tests 232} 233build_java_oracle7() { 234 use_java oracle7 235 build_java oracle7 236} 237build_java_compatibility() { 238 use_java jdk7 239 internal_build_cpp 240 # Use the unit-tests extracted from 2.5.0 to test the compatibility between 241 # 3.0.0-beta-4 and the current version. 242 cd java/compatibility_tests/v2.5.0 243 ./test.sh 3.0.0-beta-4 244 245 # Test the last released and current version. 246 ./test.sh $LAST_RELEASED 247} 248build_java_linkage_monitor() { 249 # Linkage Monitor checks compatibility with other Google libraries 250 # https://github.com/GoogleCloudPlatform/cloud-opensource-java/tree/master/linkage-monitor 251 252 use_java jdk8 253 internal_build_cpp 254 255 # Linkage Monitor uses $HOME/.m2 local repository 256 MVN="mvn -e -B -Dhttps.protocols=TLSv1.2" 257 cd java 258 # Installs the snapshot version locally 259 $MVN install -Dmaven.test.skip=true 260 261 # Linkage Monitor uses the snapshot versions installed in $HOME/.m2 to verify compatibility 262 JAR=linkage-monitor-latest-all-deps.jar 263 curl -v -O "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/${JAR}" 264 # Fails if there's new linkage errors compared with baseline 265 java -jar $JAR com.google.cloud:libraries-bom 266} 267 268build_objectivec_ios() { 269 # Reused the build script that takes care of configuring and ensuring things 270 # are up to date. The OS X test runs the objc conformance test, so skip it 271 # here. 272 objectivec/DevTools/full_mac_build.sh \ 273 --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance "$@" 274} 275 276build_objectivec_ios_debug() { 277 build_objectivec_ios --skip-xcode-release 278} 279 280build_objectivec_ios_release() { 281 build_objectivec_ios --skip-xcode-debug 282} 283 284build_objectivec_osx() { 285 # Reused the build script that takes care of configuring and ensuring things 286 # are up to date. 287 objectivec/DevTools/full_mac_build.sh \ 288 --core-only --skip-xcode-ios --skip-xcode-tvos 289} 290 291build_objectivec_tvos() { 292 # Reused the build script that takes care of configuring and ensuring things 293 # are up to date. The OS X test runs the objc conformance test, so skip it 294 # here. 295 objectivec/DevTools/full_mac_build.sh \ 296 --core-only --skip-xcode-ios --skip-xcode-osx --skip-objc-conformance "$@" 297} 298 299build_objectivec_tvos_debug() { 300 build_objectivec_tvos --skip-xcode-release 301} 302 303build_objectivec_tvos_release() { 304 build_objectivec_tvos --skip-xcode-debug 305} 306 307build_objectivec_cocoapods_integration() { 308 objectivec/Tests/CocoaPods/run_tests.sh 309} 310 311build_python() { 312 internal_build_cpp 313 cd python 314 if [ $(uname -s) == "Linux" ]; then 315 envlist=py\{27,33,34,35,36\}-python 316 else 317 envlist=py\{27,36\}-python 318 fi 319 tox -e $envlist 320 cd .. 321} 322 323build_python_version() { 324 internal_build_cpp 325 cd python 326 envlist=$1 327 tox -e $envlist 328 cd .. 329} 330 331build_python27() { 332 build_python_version py27-python 333} 334 335build_python33() { 336 build_python_version py33-python 337} 338 339build_python34() { 340 build_python_version py34-python 341} 342 343build_python35() { 344 build_python_version py35-python 345} 346 347build_python36() { 348 build_python_version py36-python 349} 350 351build_python37() { 352 build_python_version py37-python 353} 354 355build_python38() { 356 build_python_version py38-python 357} 358 359build_python_cpp() { 360 internal_build_cpp 361 export LD_LIBRARY_PATH=../src/.libs # for Linux 362 export DYLD_LIBRARY_PATH=../src/.libs # for OS X 363 cd python 364 if [ $(uname -s) == "Linux" ]; then 365 envlist=py\{27,33,34,35,36\}-cpp 366 else 367 envlist=py\{27,36\}-cpp 368 fi 369 tox -e $envlist 370 cd .. 371} 372 373build_python_cpp_version() { 374 internal_build_cpp 375 export LD_LIBRARY_PATH=../src/.libs # for Linux 376 export DYLD_LIBRARY_PATH=../src/.libs # for OS X 377 cd python 378 envlist=$1 379 tox -e $envlist 380 cd .. 381} 382 383build_python27_cpp() { 384 build_python_cpp_version py27-cpp 385} 386 387build_python33_cpp() { 388 build_python_cpp_version py33-cpp 389} 390 391build_python34_cpp() { 392 build_python_cpp_version py34-cpp 393} 394 395build_python35_cpp() { 396 build_python_cpp_version py35-cpp 397} 398 399build_python36_cpp() { 400 build_python_cpp_version py36-cpp 401} 402 403build_python37_cpp() { 404 build_python_cpp_version py37-cpp 405} 406 407build_python38_cpp() { 408 build_python_cpp_version py38-cpp 409} 410 411build_python_compatibility() { 412 internal_build_cpp 413 # Use the unit-tests extracted from 2.5.0 to test the compatibility. 414 cd python/compatibility_tests/v2.5.0 415 # Test between 2.5.0 and the current version. 416 ./test.sh 2.5.0 417 # Test between 3.0.0-beta-1 and the current version. 418 ./test.sh 3.0.0-beta-1 419 420 # Test between last released and current version. 421 ./test.sh $LAST_RELEASED 422} 423 424build_ruby23() { 425 internal_build_cpp # For conformance tests. 426 cd ruby && bash travis-test.sh ruby-2.3.8 && cd .. 427} 428build_ruby24() { 429 internal_build_cpp # For conformance tests. 430 cd ruby && bash travis-test.sh ruby-2.4 && cd .. 431} 432build_ruby25() { 433 internal_build_cpp # For conformance tests. 434 cd ruby && bash travis-test.sh ruby-2.5.1 && cd .. 435} 436build_ruby26() { 437 internal_build_cpp # For conformance tests. 438 cd ruby && bash travis-test.sh ruby-2.6.0 && cd .. 439} 440build_ruby27() { 441 internal_build_cpp # For conformance tests. 442 cd ruby && bash travis-test.sh ruby-2.7.0 && cd .. 443} 444 445build_javascript() { 446 internal_build_cpp 447 NODE_VERSION=node-v12.16.3-darwin-x64 448 NODE_TGZ="$NODE_VERSION.tar.gz" 449 pushd /tmp 450 curl -OL https://nodejs.org/dist/v12.16.3/$NODE_TGZ 451 tar zxvf $NODE_TGZ 452 export PATH=$PATH:`pwd`/$NODE_VERSION/bin 453 popd 454 cd js && npm install && npm test && cd .. 455 cd conformance && make test_nodejs && cd .. 456} 457 458use_php() { 459 VERSION=$1 460 export PATH=/usr/local/php-${VERSION}/bin:$PATH 461 internal_build_cpp 462} 463 464use_php_zts() { 465 VERSION=$1 466 export PATH=/usr/local/php-${VERSION}-zts/bin:$PATH 467 internal_build_cpp 468} 469 470build_php5.5() { 471 use_php 5.5 472 473 pushd php 474 rm -rf vendor 475 composer update 476 composer test 477 popd 478 (cd conformance && make test_php) 479} 480 481build_php5.6() { 482 use_php 5.6 483 pushd php 484 rm -rf vendor 485 composer update 486 composer test 487 popd 488 (cd conformance && make test_php) 489} 490 491build_php5.6_mac() { 492 # Install PHP 493 curl -s https://php-osx.liip.ch/install.sh | bash -s 5.6 494 PHP_FOLDER=`find /usr/local -type d -name "php5-5.6*"` # The folder name may change upon time 495 test ! -z "$PHP_FOLDER" 496 export PATH="$PHP_FOLDER/bin:$PATH" 497 498 internal_build_cpp 499 500 # Run pure-PHP tests only. 501 pushd php 502 rm -rf vendor 503 composer update 504 composer test 505 popd 506 (cd conformance && make test_php) 507} 508 509build_php7.0() { 510 use_php 7.0 511 pushd php 512 rm -rf vendor 513 composer update 514 composer test 515 popd 516 (cd conformance && make test_php) 517} 518 519build_php7.0_c() { 520 IS_64BIT=$1 521 use_php 7.0 522 php/tests/test.sh 523 pushd conformance 524 if [ "$IS_64BIT" = "true" ] 525 then 526 make test_php_c 527 else 528 make test_php_c_32 529 fi 530 popd 531} 532 533build_php7.0_mixed() { 534 use_php 7.0 535 pushd php 536 rm -rf vendor 537 composer update 538 tests/compile_extension.sh 539 tests/generate_protos.sh 540 php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit 541 popd 542} 543 544build_php7.0_zts_c() { 545 IS_64BIT=$1 546 use_php_zts 7.0 547 php/tests/test.sh 548 pushd conformance 549 if [ "$IS_64BIT" = "true" ] 550 then 551 make test_php_c 552 else 553 make test_php_c_32 554 fi 555 popd 556} 557 558build_php7.0_mac() { 559 internal_build_cpp 560 # Install PHP 561 curl -s https://php-osx.liip.ch/install.sh | bash -s 7.0 562 PHP_FOLDER=`find /usr/local -type d -name "php5-7.0*"` # The folder name may change upon time 563 test ! -z "$PHP_FOLDER" 564 export PATH="$PHP_FOLDER/bin:$PATH" 565 566 # Install valgrind 567 echo "#! /bin/bash" > valgrind 568 chmod ug+x valgrind 569 sudo mv valgrind /usr/local/bin/valgrind 570 571 # Test 572 php/tests/test.sh 573 (cd conformance && make test_php_c) 574} 575 576build_php7.3_mac() { 577 internal_build_cpp 578 # Install PHP 579 # We can't test PHP 7.4 with these binaries yet: 580 # https://github.com/liip/php-osx/issues/276 581 curl -s https://php-osx.liip.ch/install.sh | bash -s 7.3 582 PHP_FOLDER=`find /usr/local -type d -name "php5-7.3*"` # The folder name may change upon time 583 test ! -z "$PHP_FOLDER" 584 export PATH="$PHP_FOLDER/bin:$PATH" 585 586 # Install valgrind 587 echo "#! /bin/bash" > valgrind 588 chmod ug+x valgrind 589 sudo mv valgrind /usr/local/bin/valgrind 590 591 # Test 592 php/tests/test.sh 593 (cd conformance && make test_php_c) 594} 595 596build_php_compatibility() { 597 internal_build_cpp 598 php/tests/compatibility_test.sh $LAST_RELEASED 599} 600 601build_php_multirequest() { 602 use_php 7.4 603 php/tests/multirequest.sh 604} 605 606build_php7.1() { 607 use_php 7.1 608 pushd php 609 rm -rf vendor 610 composer update 611 composer test 612 popd 613 (cd conformance && make test_php) 614} 615 616build_php7.1_c() { 617 IS_64BIT=$1 618 use_php 7.1 619 php/tests/test.sh 620 pushd conformance 621 if [ "$IS_64BIT" = "true" ] 622 then 623 make test_php_c 624 else 625 make test_php_c_32 626 fi 627 popd 628} 629 630build_php7.1_mixed() { 631 use_php 7.1 632 pushd php 633 rm -rf vendor 634 composer update 635 tests/compile_extension.sh 636 tests/generate_protos.sh 637 php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit 638 popd 639} 640 641build_php7.1_zts_c() { 642 IS_64BIT=$1 643 use_php_zts 7.1 644 php/tests/test.sh 645 pushd conformance 646 if [ "$IS_64BIT" = "true" ] 647 then 648 make test_php_c 649 else 650 make test_php_c_32 651 fi 652 popd 653} 654 655build_php7.4() { 656 use_php 7.4 657 pushd php 658 rm -rf vendor 659 composer update 660 composer test 661 popd 662 (cd conformance && make test_php) 663} 664 665build_php7.4_c() { 666 IS_64BIT=$1 667 use_php 7.4 668 php/tests/test.sh 669 pushd conformance 670 if [ "$IS_64BIT" = "true" ] 671 then 672 make test_php_c 673 else 674 make test_php_c_32 675 fi 676 popd 677} 678 679build_php7.4_mixed() { 680 use_php 7.4 681 pushd php 682 rm -rf vendor 683 composer update 684 tests/compile_extension.sh 685 tests/generate_protos.sh 686 php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit 687 popd 688} 689 690build_php7.4_zts_c() { 691 IS_64BIT=$1 692 use_php_zts 7.4 693 php/tests/test.sh 694 pushd conformance 695 if [ "$IS_64BIT" = "true" ] 696 then 697 make test_php_c 698 else 699 make test_php_c_32 700 fi 701 popd 702} 703 704build_php8.0() { 705 use_php 8.0 706 pushd php 707 rm -rf vendor 708 composer update 709 composer test 710 popd 711 (cd conformance && make test_php) 712} 713 714build_php8.0_c() { 715 IS_64BIT=$1 716 use_php 8.0 717 php/tests/test.sh 718 pushd conformance 719 if [ "$IS_64BIT" = "true" ] 720 then 721 make test_php_c 722 else 723 make test_php_c_32 724 fi 725 popd 726} 727 728build_php8.0_c_64() { 729 build_php8.0_c true 730} 731 732build_php8.0_mixed() { 733 use_php 8.0 734 pushd php 735 rm -rf vendor 736 composer update 737 tests/compile_extension.sh 738 tests/generate_protos.sh 739 php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit 740 popd 741} 742 743build_php8.0_all() { 744 build_php8.0 745 build_php8.0_c_64 746 build_php8.0_mixed 747} 748 749build_php_all_32() { 750 build_php5.5 751 build_php5.6 752 build_php7.0 753 build_php7.1 754 build_php7.4 755 build_php7.0_c $1 756 build_php7.1_c $1 757 build_php7.4_c $1 758 build_php7.0_mixed 759 build_php7.1_mixed 760 build_php7.4_mixed 761 build_php7.0_zts_c $1 762 build_php7.1_zts_c $1 763 build_php7.4_zts_c $1 764} 765 766build_php_all() { 767 build_php_all_32 true 768 build_php_multirequest 769 build_php_compatibility 770} 771 772build_benchmark() { 773 use_php 7.2 774 cd kokoro/linux/benchmark && ./run.sh 775} 776 777# -------- main -------- 778 779if [ "$#" -ne 1 ]; then 780 echo " 781Usage: $0 { cpp | 782 cpp_distcheck | 783 csharp | 784 java_jdk7 | 785 java_oracle7 | 786 java_compatibility | 787 java_linkage_monitor | 788 objectivec_ios | 789 objectivec_ios_debug | 790 objectivec_ios_release | 791 objectivec_osx | 792 objectivec_tvos | 793 objectivec_tvos_debug | 794 objectivec_tvos_release | 795 objectivec_cocoapods_integration | 796 python | 797 python_cpp | 798 python_compatibility | 799 ruby23 | 800 ruby24 | 801 ruby25 | 802 ruby26 | 803 ruby27 | 804 jruby | 805 ruby_all | 806 php5.5 | 807 php5.6 | 808 php7.0 | 809 php7.0_c | 810 php_compatibility | 811 php7.1 | 812 php7.1_c | 813 php_all | 814 dist_install | 815 benchmark) 816" 817 exit 1 818fi 819 820set -e # exit immediately on error 821set -x # display all commands 822cd $(dirname $0) 823eval "build_$1" 824