1#!/usr/bin/env python3 2# Copyright 2022 The gRPC Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Helps with running bazel with extra settings to generate structured test reports in CI.""" 16 17import argparse 18import os 19import platform 20import sys 21import uuid 22 23_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../../..")) 24os.chdir(_ROOT) 25 26# How long to sleep before querying Resultstore API and uploading to bigquery 27# (to let ResultStore finish writing results from the bazel invocation that has 28# just finished). 29_UPLOAD_RBE_RESULTS_DELAY_SECONDS = 60 30 31 32def _platform_string(): 33 """Detect current platform""" 34 if platform.system() == "Windows": 35 return "windows" 36 elif platform.system()[:7] == "MSYS_NT": 37 return "windows" 38 elif platform.system() == "Darwin": 39 return "mac" 40 elif platform.system() == "Linux": 41 return "linux" 42 else: 43 return "posix" 44 45 46def _append_to_kokoro_bazel_invocations(invocation_id: str) -> None: 47 """Kokoro can display "Bazel" result link on kokoro jobs if told so.""" 48 # to get "bazel" link for kokoro build, we need to upload 49 # the "bazel_invocation_ids" file with bazel invocation ID as artifact. 50 kokoro_artifacts_dir = os.getenv("KOKORO_ARTIFACTS_DIR") 51 if kokoro_artifacts_dir: 52 # append the bazel invocation UUID to the bazel_invocation_ids file. 53 with open( 54 os.path.join(kokoro_artifacts_dir, "bazel_invocation_ids"), "a" 55 ) as f: 56 f.write(invocation_id + "\n") 57 print( 58 'Added invocation ID %s to kokoro "bazel_invocation_ids" artifact' 59 % invocation_id, 60 file=sys.stderr, 61 ) 62 else: 63 print( 64 'Skipped adding invocation ID %s to kokoro "bazel_invocation_ids"' 65 " artifact" % invocation_id, 66 file=sys.stderr, 67 ) 68 pass 69 70 71def _generate_junit_report_string( 72 report_suite_name: str, invocation_id: str, success: bool 73) -> None: 74 """Generate sponge_log.xml formatted report, that will make the bazel invocation reachable as a target in resultstore UI / sponge.""" 75 bazel_invocation_url = ( 76 "https://source.cloud.google.com/results/invocations/%s" % invocation_id 77 ) 78 package_name = report_suite_name 79 # set testcase name to invocation URL. That way, the link will be displayed in some form 80 # resultstore UI and sponge even in case the bazel invocation succeeds. 81 testcase_name = bazel_invocation_url 82 if success: 83 # unfortunately, neither resultstore UI nor sponge display the "system-err" output (or any other tags) 84 # on a passing test case. But at least we tried. 85 test_output_tag = ( 86 "<system-err>PASSED. See invocation results here: %s</system-err>" 87 % bazel_invocation_url 88 ) 89 else: 90 # The failure output will be displayes in both resultstore UI and sponge when clicking on the failing testcase. 91 test_output_tag = ( 92 '<failure message="Failure">FAILED. See bazel invocation results' 93 " here: %s</failure>" % bazel_invocation_url 94 ) 95 96 lines = [ 97 "<testsuites>", 98 '<testsuite id="1" name="%s" package="%s">' 99 % (report_suite_name, package_name), 100 '<testcase name="%s">' % testcase_name, 101 test_output_tag, 102 "</testcase></testsuite>", 103 "</testsuites>", 104 ] 105 return "\n".join(lines) 106 107 108def _create_bazel_wrapper( 109 report_path: str, 110 report_suite_name: str, 111 invocation_id: str, 112 upload_results: bool, 113) -> None: 114 """Create a "bazel wrapper" script that will execute bazel with extra settings and postprocessing.""" 115 116 os.makedirs(report_path, exist_ok=True) 117 118 bazel_wrapper_filename = os.path.join(report_path, "bazel_wrapper") 119 bazel_wrapper_bat_filename = bazel_wrapper_filename + ".bat" 120 bazel_rc_filename = os.path.join(report_path, "bazel_wrapper.bazelrc") 121 122 # put xml reports in a separate directory if requested by GRPC_TEST_REPORT_BASE_DIR 123 report_base_dir = os.getenv("GRPC_TEST_REPORT_BASE_DIR", None) 124 xml_report_path = os.path.abspath( 125 os.path.join(report_base_dir, report_path) 126 if report_base_dir 127 else report_path 128 ) 129 os.makedirs(xml_report_path, exist_ok=True) 130 131 failing_report_filename = os.path.join(xml_report_path, "sponge_log.xml") 132 success_report_filename = os.path.join( 133 xml_report_path, "success_log_to_rename.xml" 134 ) 135 136 # invoking "bash" explicitly is important for the workspace status command 137 # to work on windows as well. 138 workspace_status_command = ( 139 "bash tools/remote_build/workspace_status_kokoro.sh" 140 ) 141 142 # generate RC file with the bazel flags we want to use apply. 143 # Using an RC file solves problems with flag ordering in the wrapper. 144 # (e.g. some flags need to come after the build/test command) 145 with open(bazel_rc_filename, "w") as f: 146 f.write('build --invocation_id="%s"\n' % invocation_id) 147 f.write( 148 'build --workspace_status_command="%s"\n' % workspace_status_command 149 ) 150 151 # generate "failing" and "success" report 152 # the "failing" is named as "sponge_log.xml", which is the name picked up by sponge/resultstore 153 # so the failing report will be used by default (unless we later replace the report with 154 # one that says "success"). That way if something goes wrong before bazel is run, 155 # there will at least be a "failing" target that indicates that (we really don't want silent failures). 156 with open(failing_report_filename, "w") as f: 157 f.write( 158 _generate_junit_report_string( 159 report_suite_name, invocation_id, success=False 160 ) 161 ) 162 with open(success_report_filename, "w") as f: 163 f.write( 164 _generate_junit_report_string( 165 report_suite_name, invocation_id, success=True 166 ) 167 ) 168 169 # generate the bazel wrapper for linux/macos 170 with open(bazel_wrapper_filename, "w") as f: 171 intro_lines = [ 172 "#!/bin/bash", 173 "set -ex", 174 "", 175 'tools/bazel --bazelrc="%s" "$@" || FAILED=true' 176 % bazel_rc_filename, 177 "", 178 ] 179 180 if upload_results: 181 upload_results_lines = [ 182 "sleep %s" % _UPLOAD_RBE_RESULTS_DELAY_SECONDS, 183 "PYTHONHTTPSVERIFY=0 python3" 184 " ./tools/run_tests/python_utils/upload_rbe_results.py" 185 ' --invocation_id="%s"' % invocation_id, 186 "", 187 ] 188 else: 189 upload_results_lines = [] 190 191 outro_lines = [ 192 'if [ "$FAILED" != "" ]', 193 "then", 194 " exit 1", 195 "else", 196 ( 197 " # success: plant the pre-generated xml report that says" 198 ' "success"' 199 ), 200 " mv -f %s %s" 201 % (success_report_filename, failing_report_filename), 202 "fi", 203 ] 204 205 lines = [ 206 line + "\n" 207 for line in intro_lines + upload_results_lines + outro_lines 208 ] 209 f.writelines(lines) 210 os.chmod(bazel_wrapper_filename, 0o775) # make the unix wrapper executable 211 212 # generate bazel wrapper for windows 213 with open(bazel_wrapper_bat_filename, "w") as f: 214 intro_lines = [ 215 "@echo on", 216 "", 217 'bazel --bazelrc="%s" %%*' % bazel_rc_filename, 218 "set BAZEL_EXITCODE=%errorlevel%", 219 "", 220 ] 221 222 if upload_results: 223 upload_results_lines = [ 224 "sleep %s" % _UPLOAD_RBE_RESULTS_DELAY_SECONDS, 225 "python3 tools/run_tests/python_utils/upload_rbe_results.py" 226 ' --invocation_id="%s" || exit /b 1' % invocation_id, 227 "", 228 ] 229 else: 230 upload_results_lines = [] 231 232 outro_lines = [ 233 "if %BAZEL_EXITCODE% == 0 (", 234 ( 235 " @rem success: plant the pre-generated xml report that says" 236 ' "success"' 237 ), 238 " mv -f %s %s" 239 % (success_report_filename, failing_report_filename), 240 ")", 241 "exit /b %BAZEL_EXITCODE%", 242 ] 243 244 lines = [ 245 line + "\n" 246 for line in intro_lines + upload_results_lines + outro_lines 247 ] 248 f.writelines(lines) 249 250 print("Bazel invocation ID: %s" % invocation_id, file=sys.stderr) 251 print( 252 "Upload test results to BigQuery after bazel runs: %s" % upload_results, 253 file=sys.stderr, 254 ) 255 print( 256 "Generated bazel wrapper: %s" % bazel_wrapper_filename, file=sys.stderr 257 ) 258 print( 259 "Generated bazel wrapper: %s" % bazel_wrapper_bat_filename, 260 file=sys.stderr, 261 ) 262 263 264if __name__ == "__main__": 265 # parse command line 266 argp = argparse.ArgumentParser( 267 description=( 268 "Generate bazel wrapper to help with bazel test reports in CI." 269 ) 270 ) 271 argp.add_argument( 272 "--report_path", 273 required=True, 274 type=str, 275 help=( 276 "Path under which the bazel wrapper and other files are going to be" 277 " generated" 278 ), 279 ) 280 argp.add_argument( 281 "--report_suite_name", 282 default="bazel_invocations", 283 type=str, 284 help="Test suite name to use in generated XML report", 285 ) 286 args = argp.parse_args() 287 288 # generate new bazel invocation ID 289 invocation_id = str(uuid.uuid4()) 290 291 report_path = args.report_path 292 report_suite_name = args.report_suite_name 293 upload_results = True if os.getenv("UPLOAD_TEST_RESULTS") else False 294 295 _append_to_kokoro_bazel_invocations(invocation_id) 296 _create_bazel_wrapper( 297 report_path, report_suite_name, invocation_id, upload_results 298 ) 299