1#!/usr/bin/env bash 2# Copyright 2021 The gRPC Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15set -ex 16 17# Enter the gRPC repo root. 18cd "$(dirname "$0")/../../.." 19 20source tools/internal_ci/helper_scripts/prepare_build_linux_rc 21 22# Environment variables to select repos and branches for various repos. 23# You can edit these lines if you want to run from a fork. 24GRPC_CORE_REPO=grpc/grpc 25GRPC_CORE_GITREF=master 26GRPC_DOTNET_REPO=grpc/grpc-dotnet 27GRPC_DOTNET_GITREF=master 28GRPC_GO_REPO=grpc/grpc-go 29GRPC_GO_GITREF=master 30GRPC_JAVA_REPO=grpc/grpc-java 31GRPC_JAVA_GITREF=master 32GRPC_NODE_REPO=grpc/grpc-node 33GRPC_NODE_GITREF=master 34TEST_INFRA_REPO=grpc/test-infra 35TEST_INFRA_GITREF=master 36 37# Set up environment variables. 38LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" 39# BEGIN differentiate experimental configuration from master configuration. 40if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro && "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then 41 # Use "official" BQ tables only for builds initiated by Kokoro and running 42 # from grpc/grpc. These results show up in the "official" public dashboard. 43 BIGQUERY_TABLE_8CORE=e2e_benchmarks.ci_master_results_8core 44 BIGQUERY_TABLE_32CORE=e2e_benchmarks.ci_master_results_32core 45else 46 # Use experimental BQ tables otherwise. 47 BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results 48 BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core 49fi 50# END differentiate experimental configuration from master configuration. 51CLOUD_LOGGING_URL="https://source.cloud.google.com/results/invocations/${KOKORO_BUILD_ID}" 52PREBUILT_IMAGE_PREFIX="us-docker.pkg.dev/grpc-testing/e2etest-prebuilt" 53UNIQUE_IDENTIFIER="$(date +%Y%m%d%H%M%S)" 54ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" 55# Head of the workspace checked out by Kokoro. 56GRPC_COMMIT="$(git show --format="%H" --no-patch)" 57# Prebuilt workers for core languages are always built from grpc/grpc. 58if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then 59 GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" 60else 61 GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" 62fi 63 64GRPC_DOTNET_COMMIT="$(git ls-remote "https://github.com/${GRPC_DOTNET_REPO}.git" "${GRPC_DOTNET_GITREF}" | cut -f1)" 65GRPC_GO_COMMIT="$(git ls-remote "https://github.com/${GRPC_GO_REPO}.git" "${GRPC_GO_GITREF}" | cut -f1)" 66GRPC_JAVA_COMMIT="$(git ls-remote "https://github.com/${GRPC_JAVA_REPO}.git" "${GRPC_JAVA_GITREF}" | cut -f1)" 67GRPC_NODE_COMMIT="$(git ls-remote "https://github.com/${GRPC_NODE_REPO}.git" "${GRPC_NODE_GITREF}" | cut -f1)" 68# Kokoro jobs run on dedicated pools. 69DRIVER_POOL=drivers-ci 70WORKER_POOL_8CORE=workers-c2-8core-ci 71# c2-standard-30 is the closest machine spec to 32 core there is 72WORKER_POOL_32CORE=workers-c2-30core-ci 73# Prefix for log URLs in cnsviewer. 74LOG_URL_PREFIX="http://cnsviewer/placer/prod/home/kokoro-dedicated/build_artifacts/${KOKORO_BUILD_ARTIFACTS_SUBDIR}/github/grpc/" 75 76# This is to ensure we can push and pull images from GCR and Artifact Registry. 77# We do not necessarily need it to run load tests, but will need it when we 78# employ pre-built images in the optimization. 79gcloud auth configure-docker --quiet 80gcloud auth configure-docker "${PREBUILT_IMAGE_PREFIX%%/*}" --quiet 81 82# Connect to benchmarks-prod2 cluster. 83gcloud config set project grpc-testing 84gcloud container clusters get-credentials benchmarks-prod2 \ 85 --zone us-central1-b --project grpc-testing 86 87# Clone test-infra repository and build all tools. 88mkdir ../test-infra 89pushd ../test-infra 90git clone "https://github.com/${TEST_INFRA_REPO}.git" . 91git checkout "${TEST_INFRA_GITREF}" 92make all-tools 93popd 94 95# Build test configurations. 96buildConfigs() { 97 local -r pool="$1" 98 local -r table="$2" 99 shift 2 100 tools/run_tests/performance/loadtest_config.py "$@" \ 101 -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ 102 -s driver_pool="${DRIVER_POOL}" -s driver_image= \ 103 -s client_pool="${pool}" -s server_pool="${pool}" \ 104 -s big_query_table="${table}" -s timeout_seconds=900 \ 105 -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ 106 -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ 107 -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ 108 -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ 109 -a ci_jobName="${KOKORO_JOB_NAME}" \ 110 -a ci_gitCommit="${GRPC_COMMIT}" \ 111 -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ 112 -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ 113 -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ 114 -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ 115 -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ 116 --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ 117 -a pool="${pool}" --category=scalable \ 118 --allow_client_language=c++ --allow_server_language=c++ \ 119 --allow_server_language=node \ 120 -o "loadtest_with_prebuilt_workers_${pool}.yaml" 121} 122 123# Regex to disable specific tests. 124# https://stackoverflow.com/questions/406230 125disableTestsRegex() { 126 if (($# == 0)); then 127 echo '.*' 128 return 129 fi 130 local s='^((?!' 131 s+="$1" 132 shift 133 while (($# > 0)); do 134 s+='|' 135 s+="$1" 136 shift 137 done 138 s+=').)*$' 139 echo "${s}" 140} 141 142# List all languages. 143declare -A useLanguage=( 144 [c++]=1 145 [dotnet]=1 146 [go]=1 147 [java]=1 148 [node]=1 149 [python]=1 150 [ruby]=1 151) 152 153# Disable specific languages. 154declare -a disabledLanguages=( 155 # Add a language here to disable it. 156) 157for language in "${disabledLanguages[@]}"; do 158 unset "useLanguage[${language}]" 159done 160 161# Add arguments for languages. 162declare -a configLangArgs8core=() 163declare -a configLangArgs32core=() 164declare -a runnerLangArgs=() 165 166# c++ 167if [[ -v "useLanguage[c++]" ]]; then 168 configLangArgs8core+=(-l c++) 169 configLangArgs32core+=(-l c++) 170 runnerLangArgs+=(-l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") 171fi 172 173# dotnet 174if [[ -v "useLanguage[dotnet]" ]]; then 175 configLangArgs8core+=(-l dotnet) 176 configLangArgs32core+=(-l dotnet) 177 runnerLangArgs+=(-l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}") 178fi 179 180# go 181if [[ -v "useLanguage[go]" ]]; then 182 configLangArgs8core+=(-l go) 183 configLangArgs32core+=(-l go) 184 runnerLangArgs+=(-l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}") 185fi 186 187# java 188if [[ -v "useLanguage[java]" ]]; then 189 configLangArgs8core+=(-l java) 190 configLangArgs32core+=(-l java) 191 runnerLangArgs+=(-l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}") 192fi 193 194# node 195if [[ -v "useLanguage[node]" ]]; then 196 configLangArgs8core+=(-l node) # 8-core only. 197 runnerLangArgs+=(-l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}") 198fi 199 200# python 201if [[ -v "useLanguage[python]" ]]; then 202 configLangArgs8core+=(-l python) # 8-core only. 203 runnerLangArgs+=(-l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") 204fi 205 206# ruby 207if [[ -v "useLanguage[ruby]" ]]; then 208 configLangArgs8core+=(-l ruby) # 8-core only. 209 runnerLangArgs+=(-l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") 210fi 211 212# Disable broken tests by regex. 213# The test disabled here hangs on 8 cores. The result of this test is not 214# displayed on the public dashboard. The test runs and passes on the 30-core 215# ("32core") node pool. This can be considered a permanent fix, selectively 216# removing an unnecessary test and allowing the test run to become green. 217# IMPORTANT: Scenario names are case-sensitive. 218declare -a disabledTests8core=( 219 cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_insecure 220) 221declare -a disabledTests32core=() 222 223# Arguments to disable tests. 224regexArgs8core=(-r "$(disableTestsRegex "${disabledTests8core[@]}")") 225regexArgs32core=(-r "$(disableTestsRegex "${disabledTests32core[@]}")") 226 227buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${configLangArgs8core[@]}" "${regexArgs8core[@]}" 228buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${configLangArgs32core[@]}" "${regexArgs32core[@]}" 229 230# Delete prebuilt images on exit. 231deleteImages() { 232 echo "deleting images on exit" 233 ../test-infra/bin/delete_prebuilt_workers \ 234 -p "${PREBUILT_IMAGE_PREFIX}" \ 235 -t "${UNIQUE_IDENTIFIER}" 236} 237trap deleteImages EXIT 238 239# Build and push prebuilt images for running tests. 240time ../test-infra/bin/prepare_prebuilt_workers "${runnerLangArgs[@]}" \ 241 -p "${PREBUILT_IMAGE_PREFIX}" \ 242 -t "${UNIQUE_IDENTIFIER}" \ 243 -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" 244 245# Run tests. 246time ../test-infra/bin/runner \ 247 -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ 248 -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ 249 -log-url-prefix "${LOG_URL_PREFIX}" \ 250 -polling-interval 5s \ 251 -delete-successful-tests \ 252 -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ 253 -o "runner/sponge_log.xml" 254