1#!/usr/bin/env vpython3 2# Copyright 2017 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Runs telemetry benchmarks and gtest perf tests. 6 7If optional argument --isolated-script-test-output=[FILENAME] is passed 8to the script, json is written to that file in the format detailed in 9//docs/testing/json-test-results-format.md. 10 11If optional argument --isolated-script-test-filter=[TEST_NAMES] is passed to 12the script, it should be a double-colon-separated ("::") list of test names, 13to run just that subset of tests. 14 15This script is intended to be the base command invoked by the isolate, 16followed by a subsequent Python script. It could be generalized to 17invoke an arbitrary executable. 18It currently runs several benchmarks. The benchmarks it will execute are 19based on the shard it is running on and the sharding_map_path. 20 21If this is executed with a gtest perf test, the flag --non-telemetry 22has to be passed in to the script so the script knows it is running 23an executable and not the run_benchmark command. 24 25This script merges test results from all the benchmarks into the one 26output.json file. The test results and perf results are also put in separate 27directories per benchmark. Two files will be present in each directory; 28perf_results.json, which is the perf specific results (with unenforced format, 29could be histogram or graph json), and test_results.json. 30 31TESTING: 32To test changes to this script, please run unit tests: 33$ cd testing/scripts 34$ python3 -m unittest run_performance_tests_unittest.py 35 36Run end-to-end tests: 37$ cd tools/perf 38$ ./run_tests ScriptsSmokeTest.testRunPerformanceTests 39""" 40 41import argparse 42from collections import OrderedDict 43import json 44import os 45import pathlib 46import shutil 47import sys 48import time 49import tempfile 50import traceback 51 52import six 53 54import requests 55 56import common 57 58CHROMIUM_SRC_DIR = pathlib.Path(__file__).absolute().parents[2] 59RELEASE_DIR = CHROMIUM_SRC_DIR / 'out/Release' 60 61PERF_DIR = CHROMIUM_SRC_DIR / 'tools/perf' 62sys.path.append(str(PERF_DIR)) 63# //tools/perf imports. 64if (PERF_DIR / 'crossbench_result_converter.py').exists(): 65 # Optional import needed to run crossbench. 66 import crossbench_result_converter 67else: 68 print('Optional crossbench_result_converter not available.') 69import generate_legacy_perf_dashboard_json 70from core import path_util 71 72PERF_CORE_DIR = PERF_DIR / 'core' 73sys.path.append(str(PERF_CORE_DIR)) 74# //tools/perf/core imports. 75import results_merger 76 77sys.path.append(str(CHROMIUM_SRC_DIR / 'testing')) 78# //testing imports. 79import xvfb 80import test_env 81 82THIRD_PARTY_DIR = CHROMIUM_SRC_DIR / 'third_party' 83CATAPULT_DIR = THIRD_PARTY_DIR / 'catapult' 84TELEMETRY_DIR = CATAPULT_DIR / 'telemetry' 85# //third_party/catapult/telemetry imports. 86if TELEMETRY_DIR.exists() and (CATAPULT_DIR / 'common').exists(): 87 # Telemetry is required on perf infra, but not present on some environments. 88 sys.path.append(str(TELEMETRY_DIR)) 89 from telemetry.internal.browser import browser_finder 90 from telemetry.internal.browser import browser_options 91 from telemetry.core import util 92 from telemetry.internal.util import binary_manager 93else: 94 print('Optional telemetry library not available.') 95 96SHARD_MAPS_DIR = CHROMIUM_SRC_DIR / 'tools/perf/core/shard_maps' 97CROSSBENCH_TOOL = CHROMIUM_SRC_DIR / 'third_party/crossbench/cb.py' 98ADB_TOOL = THIRD_PARTY_DIR / 'catapult/devil/bin/deps/linux2/x86_64/bin/adb' 99GSUTIL_DIR = THIRD_PARTY_DIR / 'catapult/third_party/gsutil' 100PAGE_SETS_DATA = CHROMIUM_SRC_DIR / 'tools/perf/page_sets/data' 101PERF_TOOLS = ['benchmarks', 'executables', 'crossbench'] 102 103# See https://crbug.com/923564. 104# We want to switch over to using histograms for everything, but converting from 105# the format output by gtest perf tests to histograms has introduced several 106# problems. So, only perform the conversion on tests that are whitelisted and 107# are okay with potentially encountering issues. 108GTEST_CONVERSION_WHITELIST = [ 109 'angle_perftests', 110 'base_perftests', 111 'blink_heap_perftests', 112 'blink_platform_perftests', 113 'cc_perftests', 114 'components_perftests', 115 'command_buffer_perftests', 116 'dawn_perf_tests', 117 'gpu_perftests', 118 'load_library_perf_tests', 119 'net_perftests', 120 'browser_tests', 121 'services_perftests', 122 # TODO(jmadill): Remove once migrated. http://anglebug.com/5124 123 'standalone_angle_perftests', 124 'sync_performance_tests', 125 'tracing_perftests', 126 'views_perftests', 127 'viz_perftests', 128 'wayland_client_perftests', 129 'xr.vr.common_perftests', 130] 131 132# pylint: disable=useless-object-inheritance 133 134 135class OutputFilePaths(object): 136 """Provide paths to where results outputs should be written. 137 138 The process_perf_results.py merge script later will pull all of these 139 together, so that's why they aren't in the standard locations. Also, 140 note that because of the OBBS (One Build Bot Step), Telemetry 141 has multiple tests running on a single shard, so we need to prefix 142 these locations with a directory named by the benchmark name. 143 """ 144 145 def __init__(self, isolated_out_dir, perf_test_name): 146 self.name = perf_test_name 147 self.benchmark_path = os.path.join(isolated_out_dir, perf_test_name) 148 149 def SetUp(self): 150 if os.path.exists(self.benchmark_path): 151 shutil.rmtree(self.benchmark_path) 152 os.makedirs(self.benchmark_path) 153 return self 154 155 @property 156 def perf_results(self): 157 return os.path.join(self.benchmark_path, 'perf_results.json') 158 159 @property 160 def test_results(self): 161 return os.path.join(self.benchmark_path, 'test_results.json') 162 163 @property 164 def logs(self): 165 return os.path.join(self.benchmark_path, 'benchmark_log.txt') 166 167 @property 168 def csv_perf_results(self): 169 """Path for csv perf results. 170 171 Note that the chrome.perf waterfall uses the json histogram perf results 172 exclusively. csv_perf_results are implemented here in case a user script 173 passes --output-format=csv. 174 """ 175 return os.path.join(self.benchmark_path, 'perf_results.csv') 176 177 178def print_duration(step, start): 179 print('Duration of %s: %d seconds' % (step, time.time() - start)) 180 181 182def IsWindows(): 183 return sys.platform == 'cygwin' or sys.platform.startswith('win') 184 185 186class GtestCommandGenerator(object): 187 188 def __init__(self, 189 options, 190 override_executable=None, 191 additional_flags=None, 192 ignore_shard_env_vars=False): 193 self._options = options 194 self._override_executable = override_executable 195 self._additional_flags = additional_flags or [] 196 self._ignore_shard_env_vars = ignore_shard_env_vars 197 198 def generate(self, output_dir): 199 """Generate the command to run to start the gtest perf test. 200 201 Returns: 202 list of strings, the executable and its arguments. 203 """ 204 return ([self._get_executable()] + self._generate_filter_args() + 205 self._generate_repeat_args() + 206 self._generate_also_run_disabled_tests_args() + 207 self._generate_output_args(output_dir) + 208 self._generate_shard_args() + self._get_additional_flags()) 209 210 @property 211 def ignore_shard_env_vars(self): 212 return self._ignore_shard_env_vars 213 214 @property 215 def executable_name(self): 216 """Gets the platform-independent name of the executable.""" 217 return self._override_executable or self._options.executable 218 219 def _get_executable(self): 220 executable = str(self.executable_name) 221 if IsWindows(): 222 return r'.\%s.exe' % executable 223 return './%s' % executable 224 225 def _get_additional_flags(self): 226 return self._additional_flags 227 228 def _generate_shard_args(self): 229 """Teach the gtest to ignore the environment variables. 230 231 GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS will confuse the gtest 232 and convince it to only run some of its tests. Instead run all 233 of them. 234 """ 235 if self._ignore_shard_env_vars: 236 return ['--test-launcher-total-shards=1', '--test-launcher-shard-index=0'] 237 return [] 238 239 def _generate_filter_args(self): 240 if self._options.isolated_script_test_filter: 241 filter_list = common.extract_filter_list( 242 self._options.isolated_script_test_filter) 243 return ['--gtest_filter=' + ':'.join(filter_list)] 244 return [] 245 246 def _generate_repeat_args(self): 247 # TODO(crbug.com/40608634): Support --isolated-script-test-repeat. 248 return [] 249 250 def _generate_also_run_disabled_tests_args(self): 251 # TODO(crbug.com/40608634): Support 252 # --isolated-script-test-also-run-disabled-tests. 253 return [] 254 255 def _generate_output_args(self, output_dir): 256 output_args = [] 257 if self._options.use_gtest_benchmark_script: 258 output_args.append('--output-dir=' + output_dir) 259 # These flags are to make sure that test output perf metrics in the log. 260 if '--verbose' not in self._get_additional_flags(): 261 output_args.append('--verbose') 262 if ('--test-launcher-print-test-stdio=always' 263 not in self._get_additional_flags()): 264 output_args.append('--test-launcher-print-test-stdio=always') 265 return output_args 266 267 268def write_simple_test_results(return_code, output_filepath, benchmark_name): 269 # TODO(crbug.com/40144432): Fix to output 270 # https://chromium.googlesource.com/chromium/src/+/main/docs/testing/json_test_results_format.md 271 # for each test rather than this summary. 272 # Append the shard index to the end of the name so that the merge script 273 # doesn't blow up trying to merge unmergeable results. 274 benchmark_name += '_shard_%s' % os.environ.get('GTEST_SHARD_INDEX', '0') 275 output_json = { 276 'tests': { 277 benchmark_name: { 278 'expected': 'PASS', 279 'actual': 'FAIL' if return_code else 'PASS', 280 'is_unexpected': bool(return_code), 281 }, 282 }, 283 'interrupted': False, 284 'path_delimiter': '/', 285 'version': 3, 286 'seconds_since_epoch': time.time(), 287 'num_failures_by_type': { 288 'FAIL': 1 if return_code else 0, 289 'PASS': 0 if return_code else 1, 290 }, 291 } 292 with open(output_filepath, 'w') as fh: 293 json.dump(output_json, fh) 294 295 296def upload_simple_test_results(return_code, benchmark_name): 297 # TODO(crbug.com/40144432): Fix to upload results for each test rather than 298 # this summary. 299 try: 300 with open(os.environ['LUCI_CONTEXT']) as f: 301 sink = json.load(f)['result_sink'] 302 except KeyError: 303 return 304 305 if return_code: 306 summary = '<p>Benchmark failed with status code %d</p>' % return_code 307 else: 308 summary = '<p>Benchmark passed</p>' 309 310 result_json = { 311 'testResults': [{ 312 'testId': benchmark_name, 313 'expected': not return_code, 314 'status': 'FAIL' if return_code else 'PASS', 315 'summaryHtml': summary, 316 'tags': [{ 317 'key': 'exit_code', 318 'value': str(return_code) 319 }], 320 }] 321 } 322 323 res = requests.post( 324 url='http://%s/prpc/luci.resultsink.v1.Sink/ReportTestResults' % 325 sink['address'], 326 headers={ 327 'Content-Type': 'application/json', 328 'Accept': 'application/json', 329 'Authorization': 'ResultSink %s' % sink['auth_token'], 330 }, 331 data=json.dumps(result_json)) 332 res.raise_for_status() 333 334 335def execute_gtest_perf_test(command_generator, 336 output_paths, 337 use_xvfb=False, 338 is_unittest=False, 339 results_label=None): 340 start = time.time() 341 342 env = os.environ.copy() 343 env['CHROME_HEADLESS'] = '1' 344 # TODO(crbug.com/40153230): Some gtests do not implements the 345 # unit_test_launcher.cc. As a result, they will not respect the arguments 346 # added by _generate_shard_args() and will still use the values of 347 # GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS to run part of the tests. 348 # Removing those environment variables as a workaround. 349 if command_generator.ignore_shard_env_vars: 350 if 'GTEST_TOTAL_SHARDS' in env: 351 env.pop('GTEST_TOTAL_SHARDS') 352 if 'GTEST_SHARD_INDEX' in env: 353 env.pop('GTEST_SHARD_INDEX') 354 355 return_code = 0 356 try: 357 command = command_generator.generate(output_paths.benchmark_path) 358 if use_xvfb: 359 # When running with xvfb, we currently output both to stdout and to the 360 # file. It would be better to only output to the file to keep the logs 361 # clean. 362 return_code = xvfb.run_executable(command, 363 env, 364 stdoutfile=output_paths.logs) 365 else: 366 with open(output_paths.logs, 'w') as handle: 367 try: 368 return_code = test_env.run_command_output_to_handle(command, 369 handle, 370 env=env) 371 except OSError as e: 372 print('Command to run gtest perf test %s failed with an OSError: %s' % 373 (output_paths.name, e)) 374 return_code = 1 375 if (not os.path.exists(output_paths.perf_results) 376 and os.path.exists(output_paths.logs)): 377 # Get the correct json format from the stdout to write to the perf 378 # results file if gtest does not generate one. 379 results_processor = generate_legacy_perf_dashboard_json.\ 380 LegacyResultsProcessor() 381 graph_json_string = results_processor.GenerateJsonResults( 382 output_paths.logs) 383 with open(output_paths.perf_results, 'w') as fh: 384 fh.write(graph_json_string) 385 except Exception: # pylint: disable=broad-except 386 traceback.print_exc() 387 return_code = 1 388 if os.path.exists(output_paths.perf_results): 389 executable_name = command_generator.executable_name 390 if executable_name.startswith('bin/run_'): 391 # The executable is a wrapper used by Fuchsia. Remove the prefix to get 392 # the actual executable name. 393 executable_name = executable_name[8:] 394 if executable_name in GTEST_CONVERSION_WHITELIST: 395 with path_util.SysPath(path_util.GetTracingDir()): 396 # pylint: disable=no-name-in-module,import-outside-toplevel 397 from tracing.value import gtest_json_converter 398 # pylint: enable=no-name-in-module,import-outside-toplevel 399 gtest_json_converter.ConvertGtestJsonFile(output_paths.perf_results, 400 label=results_label) 401 else: 402 print('ERROR: gtest perf test %s did not generate perf output' % 403 output_paths.name) 404 return_code = 1 405 write_simple_test_results(return_code, output_paths.test_results, 406 output_paths.name) 407 if not is_unittest: 408 upload_simple_test_results(return_code, output_paths.name) 409 410 print_duration('executing gtest %s' % command_generator.executable_name, 411 start) 412 413 return return_code 414 415 416class _TelemetryFilterArgument(object): 417 418 def __init__(self, filter_string): 419 self.benchmark, self.story = filter_string.split('/') 420 421 422class TelemetryCommandGenerator(object): 423 424 def __init__(self, 425 benchmark, 426 options, 427 story_selection_config=None, 428 is_reference=False): 429 self.benchmark = benchmark 430 self._options = options 431 self._story_selection_config = story_selection_config 432 self._is_reference = is_reference 433 434 def generate(self, output_dir): 435 """Generate the command to run to start the benchmark. 436 437 Args: 438 output_dir: The directory to configure the command to put output files 439 into. 440 441 Returns: 442 list of strings, the executable and its arguments. 443 """ 444 return ( 445 [sys.executable] + self._options.executable.split(' ') + 446 [self.benchmark] + self._generate_filter_args() + 447 self._generate_also_run_disabled_tests_args() + 448 self._generate_output_args(output_dir) + 449 self._generate_story_selection_args() + 450 # passthrough args must be before reference args and repeat args: 451 # crbug.com/928928, crbug.com/894254#c78 452 self._get_passthrough_args() + self._generate_syslog_args() + 453 self._generate_repeat_args() + self._generate_reference_build_args() + 454 self._generate_results_label_args()) 455 456 def _get_passthrough_args(self): 457 return self._options.passthrough_args 458 459 def _generate_filter_args(self): 460 if self._options.isolated_script_test_filter: 461 filter_list = common.extract_filter_list( 462 self._options.isolated_script_test_filter) 463 filter_arguments = [_TelemetryFilterArgument(f) for f in filter_list] 464 applicable_stories = [ 465 f.story for f in filter_arguments if f.benchmark == self.benchmark 466 ] 467 # Need to convert this to a valid regex. 468 filter_regex = '(' + '|'.join(applicable_stories) + ')' 469 return ['--story-filter=' + filter_regex] 470 return [] 471 472 def _generate_repeat_args(self): 473 pageset_repeat = None 474 if self._options.isolated_script_test_repeat: 475 pageset_repeat = self._options.isolated_script_test_repeat 476 elif (self._story_selection_config is not None 477 and self._story_selection_config.get('pageset_repeat')): 478 pageset_repeat = self._story_selection_config.get('pageset_repeat') 479 480 if pageset_repeat: 481 return ['--pageset-repeat=' + str(pageset_repeat)] 482 return [] 483 484 def _generate_also_run_disabled_tests_args(self): 485 if self._options.isolated_script_test_also_run_disabled_tests: 486 return ['--also-run-disabled-tests'] 487 return [] 488 489 def _generate_output_args(self, output_dir): 490 if self._options.no_output_conversion: 491 return ['--output-format=none', '--output-dir=' + output_dir] 492 return [ 493 '--output-format=json-test-results', '--output-format=histograms', 494 '--output-dir=' + output_dir 495 ] 496 497 def _generate_story_selection_args(self): 498 """Returns arguments that limit the stories to be run inside the benchmark. 499 """ 500 selection_args = [] 501 if self._story_selection_config: 502 if 'begin' in self._story_selection_config: 503 selection_args.append('--story-shard-begin-index=%d' % 504 (self._story_selection_config['begin'])) 505 if 'end' in self._story_selection_config: 506 selection_args.append('--story-shard-end-index=%d' % 507 (self._story_selection_config['end'])) 508 if 'sections' in self._story_selection_config: 509 range_string = self._generate_story_index_ranges( 510 self._story_selection_config['sections']) 511 if range_string: 512 selection_args.append('--story-shard-indexes=%s' % range_string) 513 if self._story_selection_config.get('abridged', True): 514 selection_args.append('--run-abridged-story-set') 515 return selection_args 516 517 def _generate_syslog_args(self): 518 if self._options.per_test_logs_dir: 519 isolated_out_dir = os.path.dirname( 520 self._options.isolated_script_test_output) 521 return ['--logs-dir', os.path.join(isolated_out_dir, self.benchmark)] 522 return [] 523 524 def _generate_story_index_ranges(self, sections): 525 range_string = '' 526 for section in sections: 527 begin = section.get('begin', '') 528 end = section.get('end', '') 529 # If there only one story in the range, we only keep its index. 530 # In general, we expect either begin or end, or both. 531 if begin != '' and end != '' and end - begin == 1: 532 new_range = str(begin) 533 elif begin != '' or end != '': 534 new_range = '%s-%s' % (str(begin), str(end)) 535 else: 536 raise ValueError('Index ranges in "sections" in shard map should have' 537 'at least one of "begin" and "end": %s' % str(section)) 538 if range_string: 539 range_string += ',%s' % new_range 540 else: 541 range_string = new_range 542 return range_string 543 544 def _generate_reference_build_args(self): 545 if self._is_reference: 546 reference_browser_flag = '--browser=reference' 547 # TODO(crbug.com/40113070): Make the logic generic once more reference 548 # settings are added 549 if '--browser=android-chrome-bundle' in self._get_passthrough_args(): 550 reference_browser_flag = '--browser=reference-android-chrome-bundle' 551 return [reference_browser_flag, '--max-failures=5'] 552 return [] 553 554 def _generate_results_label_args(self): 555 if self._options.results_label: 556 return ['--results-label=' + self._options.results_label] 557 return [] 558 559 560def execute_telemetry_benchmark(command_generator, 561 output_paths, 562 use_xvfb=False, 563 return_exit_code_zero=False, 564 no_output_conversion=False): 565 start = time.time() 566 567 env = os.environ.copy() 568 env['CHROME_HEADLESS'] = '1' 569 570 return_code = 1 571 temp_dir = tempfile.mkdtemp('telemetry') 572 infra_failure = False 573 try: 574 command = command_generator.generate(temp_dir) 575 if use_xvfb: 576 # When running with xvfb, we currently output both to stdout and to the 577 # file. It would be better to only output to the file to keep the logs 578 # clean. 579 return_code = xvfb.run_executable(command, 580 env=env, 581 stdoutfile=output_paths.logs) 582 else: 583 with open(output_paths.logs, 'w') as handle: 584 return_code = test_env.run_command_output_to_handle(command, 585 handle, 586 env=env) 587 expected_results_filename = os.path.join(temp_dir, 'test-results.json') 588 if os.path.exists(expected_results_filename): 589 shutil.move(expected_results_filename, output_paths.test_results) 590 else: 591 common.write_interrupted_test_results_to(output_paths.test_results, start) 592 593 if not no_output_conversion: 594 expected_perf_filename = os.path.join(temp_dir, 'histograms.json') 595 if os.path.exists(expected_perf_filename): 596 shutil.move(expected_perf_filename, output_paths.perf_results) 597 elif return_code: 598 print(f'The benchmark failed with status code {return_code}, ' 599 'and did not produce perf results output. ' 600 'Check benchmark output for more details.') 601 else: 602 print('The benchmark returned a success status code, ' 603 'but did not product perf results output.') 604 605 csv_file_path = os.path.join(temp_dir, 'results.csv') 606 if os.path.isfile(csv_file_path): 607 shutil.move(csv_file_path, output_paths.csv_perf_results) 608 except Exception: # pylint: disable=broad-except 609 print('The following exception may have prevented the code from ' 610 'outputing structured test results and perf results output:') 611 print(traceback.format_exc()) 612 infra_failure = True 613 finally: 614 # On swarming bots, don't remove output directory, since Result Sink might 615 # still be uploading files to Result DB. Also, swarming bots automatically 616 # clean up at the end of each task. 617 if 'SWARMING_TASK_ID' not in os.environ: 618 # Add ignore_errors=True because otherwise rmtree may fail due to leaky 619 # processes of tests are still holding opened handles to files under 620 # |tempfile_dir|. For example, see crbug.com/865896 621 shutil.rmtree(temp_dir, ignore_errors=True) 622 623 print_duration('executing benchmark %s' % command_generator.benchmark, start) 624 625 if infra_failure: 626 print('There was an infrastructure error encountered during the run. ' 627 'Please check the logs above for details') 628 return 1 629 630 # Telemetry sets exit code to -1 to indicate that no stories were run. This 631 # becomes 255 on linux because linux doesn't support -1 so it does modulo: 632 # -1 % 256 == 255. 633 # TODO(crbug.com/40105219): Make 111 be the exit code that means 634 # "no stories were run.". 635 if return_code in (111, -1, 255): 636 print('Exit code %s indicates that no stories were run, so we are marking ' 637 'this as a success.' % return_code) 638 return 0 639 if return_code: 640 if return_exit_code_zero: 641 print('run_benchmark returned exit code ' + str(return_code) + 642 ' which indicates there were test failures in the run.') 643 return 0 644 return return_code 645 return 0 646 647 648def load_map_file(map_file, isolated_out_dir): 649 """Loads the shard map file and copies it to isolated_out_dir.""" 650 if not os.path.exists(map_file): 651 map_file_path = SHARD_MAPS_DIR / map_file 652 if map_file_path.exists(): 653 map_file = str(map_file_path) 654 else: 655 raise Exception(f'Test shard map file not found: {map_file_path}') 656 copy_map_file_to_out_dir(map_file, isolated_out_dir) 657 with open(map_file) as f: 658 return json.load(f) 659 660 661def load_map_string(map_string, isolated_out_dir): 662 """Loads the dynamic shard map string and writes it to isolated_out_dir.""" 663 if not map_string: 664 raise Exception('Use `--dynamic-shardmap` to pass the dynamic shard map') 665 shard_map = json.loads(map_string, object_pairs_hook=OrderedDict) 666 with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmp: 667 tmp.write(bytes(map_string, 'utf-8')) 668 tmp.close() 669 copy_map_file_to_out_dir(tmp.name, isolated_out_dir) 670 return shard_map 671 672 673def copy_map_file_to_out_dir(map_file, isolated_out_dir): 674 """Copies the sharding map file to isolated_out_dir for later collection.""" 675 if not os.path.exists(isolated_out_dir): 676 os.makedirs(isolated_out_dir) 677 shutil.copyfile(map_file, 678 os.path.join(isolated_out_dir, 'benchmarks_shard_map.json')) 679 680 681def fetch_binary_path(dependency_name, os_name='linux', arch='x86_64'): 682 if binary_manager.NeedsInit(): 683 binary_manager.InitDependencyManager(None) 684 return binary_manager.FetchPath(dependency_name, os_name=os_name, arch=arch) 685 686 687class CrossbenchTest(object): 688 """This class is for running Crossbench tests. 689 690 To run Crossbench tests, pass the relative path to `cb.py` as the 691 `executable` argument, followed by `--benchmarks` to specify the 692 target benchmark. The remaining Crossbench arguments are optional, 693 and any passed arguments are sent to the `cb.py` for executing the test. 694 695 Shard map: Use `crossbench-benchmarks` node in each shard group with a 696 dictionary of benchmark names and, optionally, a list of arguments that need 697 to pass through the `cb.py` tool. See `linux-perf-fyi_map.json` for examples. 698 699 Example: 700 ./run_performance_tests.py ../../third_party/crossbench/cb.py \ 701 --isolated-script-test-output=/tmp/crossbench/ \ 702 --benchmarks=speedometer \ 703 --browser=../../out/linux/chrome \ 704 --repeat=1 --probe='profiling' --story='jQuery.*' 705 """ 706 707 EXECUTABLE = 'cb.py' 708 OUTDIR = '--out-dir=%s/output' 709 CHROME_BROWSER = '--browser=%s' 710 ANDROID_HJSON = '{browser:"%s", driver:{type:"Android", adb_bin:"%s"}}' 711 STORY_LABEL = 'default' 712 BENCHMARK_FILESERVERS = { 713 'speedometer_3.0': 'third_party/speedometer/v3.0', 714 'speedometer_2.1': 'third_party/speedometer/v2.1', 715 'speedometer_2.0': 'third_party/speedometer/v2.0' 716 } 717 718 def __init__(self, options, isolated_out_dir): 719 self.options = options 720 self.isolated_out_dir = isolated_out_dir 721 browser_arg = self._get_browser_arg(options.passthrough_args) 722 self.is_android = self._is_android(browser_arg) 723 self._find_browser(browser_arg) 724 self.driver_path_arg = self._find_chromedriver(browser_arg) 725 self.network = self._get_network_arg(options.passthrough_args) 726 727 def _get_browser_arg(self, args): 728 browser_arg = self._get_arg(args, '--browser=', must_exists=True) 729 return browser_arg.split('=', 1)[1] 730 731 def _get_network_arg(self, args): 732 if _arg := self._get_arg(args, '--network='): 733 return [_arg] 734 if _arg := self._get_arg(args, '--fileserver'): 735 return self._create_fileserver_network(_arg) 736 if self._get_arg(args, '--wpr'): 737 return self._create_wpr_network(args) 738 if self.options.benchmarks in self.BENCHMARK_FILESERVERS: 739 # Use file server when it is available. 740 arg = '--fileserver' 741 args.append(arg) 742 return self._create_fileserver_network(arg) 743 return [] 744 745 def _create_fileserver_network(self, arg): 746 if '=' in arg: 747 fileserver_path = arg.split('=', 1)[1] 748 else: 749 benchmark = self.options.benchmarks 750 if benchmark not in self.BENCHMARK_FILESERVERS: 751 raise ValueError(f'fileserver does not support {benchmark}') 752 fileserver_path = self.BENCHMARK_FILESERVERS.get(benchmark) 753 fileserver_relative_path = str(CHROMIUM_SRC_DIR / fileserver_path) 754 # Replacing --fileserver with --network. 755 self.options.passthrough_args.remove(arg) 756 return [ 757 self._create_network_json('local', 758 path=fileserver_relative_path, 759 url='http://localhost:0') 760 ] 761 762 def _create_wpr_network(self, args): 763 wpr_arg = self._get_arg(args, '--wpr') 764 if wpr_arg and '=' in wpr_arg: 765 wpr_name = wpr_arg.split('=', 1)[1] 766 else: 767 raise ValueError('The archive file path is missing!') 768 archive = str(PAGE_SETS_DATA / wpr_name) 769 if (wpr_go := fetch_binary_path('wpr_go')) is None: 770 raise ValueError(f'wpr_go not found: {wpr_go}') 771 if wpr_arg: 772 # Replacing --wpr with --network. 773 self.options.passthrough_args.remove(wpr_arg) 774 return [self._create_network_json('wpr', path=archive, wpr_go_bin=wpr_go)] 775 776 def _create_network_json(self, config_type, path, url=None, wpr_go_bin=None): 777 network_dict = {'type': config_type} 778 network_dict['path'] = path 779 if url: 780 network_dict['url'] = url 781 if wpr_go_bin: 782 network_dict['wpr_go_bin'] = wpr_go_bin 783 network_json = json.dumps(network_dict) 784 return f'--network={network_json}' 785 786 def _get_arg(self, args, arg, must_exists=False): 787 if _args := [a for a in args if a.startswith(arg)]: 788 if len(_args) != 1: 789 raise ValueError(f'Expects exactly one {arg} on command line') 790 return _args[0] 791 if must_exists: 792 raise ValueError(f'{arg} argument is missing!') 793 return [] 794 795 def _is_android(self, browser_arg): 796 """Is the test running on an Android device. 797 798 See third_party/catapult/telemetry/telemetry/internal/backends/android_browser_backend_settings.py # pylint: disable=line-too-long 799 """ 800 return browser_arg.lower().startswith('android') 801 802 def _find_browser(self, browser_arg): 803 # Replacing --browser with the generated self.browser. 804 self.options.passthrough_args = [ 805 arg for arg in self.options.passthrough_args 806 if not arg.startswith('--browser=') 807 ] 808 if '/' in browser_arg or '\\' in browser_arg: 809 # The --browser arg looks like a path. Use it as-is. 810 self.browser = self.CHROME_BROWSER % browser_arg 811 return 812 options = browser_options.BrowserFinderOptions() 813 options.chrome_root = CHROMIUM_SRC_DIR 814 parser = options.CreateParser() 815 parser.parse_args([self.CHROME_BROWSER % browser_arg]) 816 possible_browser = browser_finder.FindBrowser(options) 817 if not possible_browser: 818 raise ValueError(f'Unable to find Chrome browser of type: {browser_arg}') 819 if self.is_android: 820 browser_app = possible_browser.settings.package 821 android_json = self.ANDROID_HJSON % (browser_app, ADB_TOOL) 822 self.browser = self.CHROME_BROWSER % android_json 823 else: 824 assert hasattr(possible_browser, 'local_executable') 825 self.browser = self.CHROME_BROWSER % possible_browser.local_executable 826 827 def _find_chromedriver(self, browser_arg): 828 browser_arg = browser_arg.lower() 829 if browser_arg == 'release_x64': 830 path = '../Release_x64' 831 elif self.is_android: 832 path = 'clang_x64' 833 else: 834 path = '.' 835 836 abspath = pathlib.Path(path).absolute() 837 if ((driver_path := (abspath / 'chromedriver')).exists() 838 or (driver_path := (abspath / 'chromedriver.exe')).exists()): 839 return [f'--driver-path={driver_path}'] 840 # Unable to find ChromeDriver, will rely on crossbench to download one. 841 return [] 842 843 def _get_default_args(self): 844 default_args = [ 845 '--no-symlinks', 846 # Required until crbug/41491492 and crbug/346323630 are fixed. 847 '--enable-features=DisablePrivacySandboxPrompts', 848 ] 849 if not self.is_android: 850 # See http://shortn/_xGSaVM9P5g 851 default_args.append('--enable-field-trial-config') 852 return default_args 853 854 def _generate_command_list(self, benchmark, benchmark_args, working_dir): 855 return (['vpython3'] + [self.options.executable] + [benchmark] + 856 ['--env-validation=throw'] + [self.OUTDIR % working_dir] + 857 [self.browser] + benchmark_args + self.driver_path_arg + 858 self.network + self._get_default_args()) 859 860 def execute_benchmark(self, 861 benchmark, 862 display_name, 863 benchmark_args, 864 is_unittest=False): 865 start = time.time() 866 867 env = os.environ.copy() 868 env['CHROME_HEADLESS'] = '1' 869 env['PATH'] = f'{GSUTIL_DIR}:' + env['PATH'] 870 871 return_code = 1 872 output_paths = OutputFilePaths(self.isolated_out_dir, display_name).SetUp() 873 infra_failure = False 874 try: 875 command = self._generate_command_list(benchmark, benchmark_args, 876 output_paths.benchmark_path) 877 if self.options.xvfb: 878 # When running with xvfb, we currently output both to stdout and to the 879 # file. It would be better to only output to the file to keep the logs 880 # clean. 881 return_code = xvfb.run_executable(command, 882 env=env, 883 stdoutfile=output_paths.logs) 884 else: 885 with open(output_paths.logs, 'w') as handle: 886 return_code = test_env.run_command_output_to_handle(command, 887 handle, 888 env=env) 889 890 if return_code == 0: 891 crossbench_result_converter.convert( 892 pathlib.Path(output_paths.benchmark_path) / 'output', 893 pathlib.Path(output_paths.perf_results), display_name, 894 self.STORY_LABEL, self.options.results_label) 895 except Exception: # pylint: disable=broad-except 896 print('The following exception may have prevented the code from ' 897 'outputing structured test results and perf results output:') 898 print(traceback.format_exc()) 899 infra_failure = True 900 901 write_simple_test_results(return_code, output_paths.test_results, 902 display_name) 903 if not is_unittest: 904 upload_simple_test_results(return_code, display_name) 905 906 print_duration(f'Executing benchmark: {benchmark}', start) 907 908 if infra_failure: 909 print('There was an infrastructure error encountered during the run. ' 910 'Please check the logs above for details') 911 return 1 912 913 if return_code and self.options.ignore_benchmark_exit_code: 914 print(f'crossbench returned exit code {return_code}' 915 ' which indicates there were test failures in the run.') 916 return 0 917 return return_code 918 919 def execute(self): 920 if not self.options.benchmarks: 921 raise Exception('Please use the --benchmarks to specify the benchmark.') 922 if ',' in self.options.benchmarks: 923 raise Exception('No support to run multiple benchmarks at this time.') 924 return self.execute_benchmark( 925 self.options.benchmarks, 926 (self.options.benchmark_display_name or self.options.benchmarks), 927 self.options.passthrough_args) 928 929 930def parse_arguments(args): 931 parser = argparse.ArgumentParser() 932 parser.add_argument('executable', help='The name of the executable to run.') 933 parser.add_argument('--isolated-script-test-output', required=True) 934 # The following two flags may be passed in sometimes by Pinpoint 935 # or by the recipe, but they don't do anything. crbug.com/927482. 936 parser.add_argument('--isolated-script-test-chartjson-output', required=False) 937 parser.add_argument('--isolated-script-test-perf-output', required=False) 938 939 parser.add_argument('--isolated-script-test-filter', type=str, required=False) 940 941 # Note that the following three arguments are only supported by Telemetry 942 # tests right now. See crbug.com/920002. 943 parser.add_argument('--isolated-script-test-repeat', type=int, required=False) 944 parser.add_argument( 945 '--isolated-script-test-launcher-retry-limit', 946 type=int, 947 required=False, 948 choices=[0]) # Telemetry does not support retries. crbug.com/894254#c21 949 parser.add_argument('--isolated-script-test-also-run-disabled-tests', 950 default=False, 951 action='store_true', 952 required=False) 953 parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') 954 parser.add_argument('--non-telemetry', 955 help='Type of perf test', 956 type=bool, 957 default=False) 958 parser.add_argument('--gtest-benchmark-name', 959 help='Name of the gtest benchmark', 960 type=str, 961 required=False) 962 parser.add_argument('--use-gtest-benchmark-script', 963 help='Whether gtest is invoked via benchmark script.', 964 default=False, 965 action='store_true') 966 967 parser.add_argument('--benchmarks', 968 help='Comma separated list of benchmark names' 969 ' to run in lieu of indexing into our benchmark bot maps', 970 required=False) 971 parser.add_argument('--benchmark-display-name', 972 help='Benchmark name displayed to the user,' 973 ' supported with crossbench only', 974 required=False) 975 # Added to address android flakiness. 976 parser.add_argument('--benchmark-max-runs', 977 help='Max number of benchmark runs until it succeeds.', 978 type=int, 979 required=False, 980 default=1) 981 # crbug.com/1236245: This allows for per-benchmark device logs. 982 parser.add_argument('--per-test-logs-dir', 983 help='Require --logs-dir args for test', 984 required=False, 985 default=False, 986 action='store_true') 987 # Some executions may have a different sharding scheme and/or set of tests. 988 # These files must live in src/tools/perf/core/shard_maps 989 parser.add_argument('--test-shard-map-filename', type=str, required=False) 990 parser.add_argument('--run-ref-build', 991 help='Run test on reference browser', 992 action='store_true') 993 parser.add_argument('--passthrough-arg', 994 help='Arguments to pass directly through to the test ' 995 'executable.', 996 action='append', 997 dest='passthrough_args', 998 default=[]) 999 parser.add_argument('--use-dynamic-shards', 1000 help='If set, use dynamic shardmap instead of the file.', 1001 action='store_true', 1002 required=False) 1003 parser.add_argument('--dynamic-shardmap', 1004 help='The dynamically generated shardmap string used to ' 1005 'replace the static shardmap file.', 1006 type=str, 1007 required=False) 1008 parser.add_argument('--ignore-benchmark-exit-code', 1009 help='If set, return an exit code 0 even if there' + 1010 ' are benchmark failures', 1011 action='store_true', 1012 required=False) 1013 parser.add_argument('--results-label', 1014 help='If set for a non-telemetry test, adds label to' + 1015 ' the result histograms.', 1016 type=str, 1017 required=False) 1018 parser.add_argument('--no-output-conversion', 1019 help='If supplied, trace conversion is not done.', 1020 action='store_true', 1021 required=False, 1022 default=False) 1023 options, leftover_args = parser.parse_known_args(args) 1024 options.passthrough_args.extend(leftover_args) 1025 return options 1026 1027 1028def _set_cwd(): 1029 """Change current working directory to build output directory. 1030 1031 On perf waterfall, the recipe sets the current working directory to the chrome 1032 build output directory. Pinpoint, on the other hand, does not know where the 1033 build output directory is, so it is hardcoded to set current working directory 1034 to out/Release. This used to be correct most of the time, but this has been 1035 changed by https://crbug.com/355218109, causing various problems (e.g., 1036 https://crbug.com/377748127). This function attempts to detect such cases and 1037 change the current working directory to chrome output directory. 1038 """ 1039 1040 # If the current directory is named out/Release and is empty, we are likely 1041 # running on Pinpoint with a wrong working directory. 1042 cwd = pathlib.Path.cwd() 1043 if list(cwd.iterdir()): 1044 return 1045 if cwd.name != 'Release' or cwd.parent.name != 'out': 1046 return 1047 1048 print(f'Current directory {cwd} is empty, attempting to find build output') 1049 candidates = [] 1050 for build_dir in util.GetBuildDirectories(): 1051 path = pathlib.Path(build_dir).resolve() 1052 if path.exists() and list(path.iterdir()): 1053 candidates.append(path) 1054 1055 if len(candidates) != 1: 1056 if not candidates: 1057 print('No build output directory found') 1058 else: 1059 print(f'Multiple build output directories found: {candidates}') 1060 raise RuntimeError( 1061 'Unable to find build output. Please change to the build output ' 1062 'directory before running this script.') 1063 1064 print(f'Changing current directory to {candidates[0]}') 1065 os.chdir(candidates[0]) 1066 1067 1068def main(sys_args): 1069 sys.stdout.reconfigure(line_buffering=True) 1070 _set_cwd() 1071 args = sys_args[1:] # Skip program name. 1072 options = parse_arguments(args) 1073 isolated_out_dir = os.path.dirname(options.isolated_script_test_output) 1074 overall_return_code = 0 1075 # This is a list of test results files to be merged into a standard 1076 # output.json file for use by infrastructure including FindIt. 1077 # This list should not contain reference build runs 1078 # since we do not monitor those. Also, merging test reference build results 1079 # with standard build results may not work properly. 1080 test_results_files = [] 1081 1082 print('Running a series of performance test subprocesses. Logs, performance\n' 1083 'results, and test results JSON will be saved in a subfolder of the\n' 1084 'isolated output directory. Inside the hash marks in the following\n' 1085 'lines is the name of the subfolder to find results in.\n') 1086 1087 if options.use_dynamic_shards: 1088 shard_map = load_map_string(options.dynamic_shardmap, isolated_out_dir) 1089 overall_return_code = _run_benchmarks_on_shardmap(shard_map, options, 1090 isolated_out_dir, 1091 test_results_files) 1092 elif options.test_shard_map_filename: 1093 shard_map = load_map_file(options.test_shard_map_filename, isolated_out_dir) 1094 overall_return_code = _run_benchmarks_on_shardmap(shard_map, options, 1095 isolated_out_dir, 1096 test_results_files) 1097 elif options.executable.endswith(CrossbenchTest.EXECUTABLE): 1098 assert options.benchmark_max_runs == 1, ( 1099 'Benchmark rerun is not supported with CrossbenchTest.') 1100 overall_return_code = CrossbenchTest(options, isolated_out_dir).execute() 1101 elif options.non_telemetry: 1102 assert options.benchmark_max_runs == 1, ( 1103 'Benchmark rerun is not supported in non telemetry tests.') 1104 benchmark_name = options.gtest_benchmark_name 1105 passthrough_args = options.passthrough_args 1106 # crbug/1146949#c15 1107 # In the case that pinpoint passes all arguments to swarming through http 1108 # request, the passthrough_args are converted into a comma-separated string. 1109 if passthrough_args and isinstance(passthrough_args, six.text_type): 1110 passthrough_args = passthrough_args.split(',') 1111 # With --non-telemetry, the gtest executable file path will be passed in as 1112 # options.executable, which is different from running on shard map. Thus, 1113 # we don't override executable as we do in running on shard map. 1114 command_generator = GtestCommandGenerator(options, 1115 additional_flags=passthrough_args, 1116 ignore_shard_env_vars=True) 1117 # Fallback to use the name of the executable if flag isn't set. 1118 # TODO(crbug.com/40588014): remove fallback logic and raise parser error if 1119 # --non-telemetry is set but --gtest-benchmark-name is not set once pinpoint 1120 # is converted to always pass --gtest-benchmark-name flag. 1121 if not benchmark_name: 1122 benchmark_name = options.executable 1123 output_paths = OutputFilePaths(isolated_out_dir, benchmark_name).SetUp() 1124 print('\n### {folder} ###'.format(folder=benchmark_name)) 1125 overall_return_code = execute_gtest_perf_test( 1126 command_generator, 1127 output_paths, 1128 options.xvfb, 1129 results_label=options.results_label) 1130 test_results_files.append(output_paths.test_results) 1131 elif options.benchmarks: 1132 benchmarks = options.benchmarks.split(',') 1133 for benchmark in benchmarks: 1134 command_generator = TelemetryCommandGenerator(benchmark, options) 1135 for run_num in range(options.benchmark_max_runs): 1136 print('\n### {folder} (attempt #{num}) ###'.format(folder=benchmark, 1137 num=run_num)) 1138 output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp() 1139 return_code = execute_telemetry_benchmark( 1140 command_generator, 1141 output_paths, 1142 options.xvfb, 1143 options.ignore_benchmark_exit_code, 1144 no_output_conversion=options.no_output_conversion) 1145 if return_code == 0: 1146 break 1147 overall_return_code = return_code or overall_return_code 1148 test_results_files.append(output_paths.test_results) 1149 if options.run_ref_build: 1150 print('Not running reference build. --run-ref-build argument is only ' 1151 'supported for sharded benchmarks. It is simple to support ' 1152 'this for unsharded --benchmarks if needed.') 1153 else: 1154 raise Exception('Telemetry tests must provide either a shard map or a ' 1155 '--benchmarks list so that we know which stories to run.') 1156 1157 # Dumping the test results. 1158 if test_results_files: 1159 test_results_list = [] 1160 for test_results_file in test_results_files: 1161 if os.path.exists(test_results_file): 1162 with open(test_results_file, 'r') as fh: 1163 test_results_list.append(json.load(fh)) 1164 merged_test_results = results_merger.merge_test_results(test_results_list) 1165 with open(options.isolated_script_test_output, 'w') as f: 1166 json.dump(merged_test_results, f) 1167 1168 return overall_return_code 1169 1170 1171def _run_benchmarks_on_shardmap(shard_map, options, isolated_out_dir, 1172 test_results_files): 1173 overall_return_code = 0 1174 # TODO(crbug.com/40631538): shard environment variables are not specified 1175 # for single-shard shard runs. 1176 if 'GTEST_SHARD_INDEX' not in os.environ and '1' in shard_map.keys(): 1177 raise Exception( 1178 'Setting GTEST_SHARD_INDEX environment variable is required ' 1179 'when you use a shard map.') 1180 shard_index = os.environ.get('GTEST_SHARD_INDEX', '0') 1181 shard_configuration = shard_map[shard_index] 1182 if not [x for x in shard_configuration if x in PERF_TOOLS]: 1183 raise Exception( 1184 f'None of {",".join(PERF_TOOLS)} presented in the shard map') 1185 if 'benchmarks' in shard_configuration: 1186 benchmarks_and_configs = shard_configuration['benchmarks'] 1187 for (benchmark, story_selection_config) in benchmarks_and_configs.items(): 1188 # Need to run the benchmark on both latest browser and reference 1189 # build. 1190 command_generator = TelemetryCommandGenerator( 1191 benchmark, options, story_selection_config=story_selection_config) 1192 for run_num in range(options.benchmark_max_runs): 1193 output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp() 1194 print('\n### {folder} (attempt #{num}) ###'.format(folder=benchmark, 1195 num=run_num)) 1196 return_code = execute_telemetry_benchmark( 1197 command_generator, 1198 output_paths, 1199 options.xvfb, 1200 options.ignore_benchmark_exit_code, 1201 no_output_conversion=options.no_output_conversion) 1202 if return_code == 0: 1203 break 1204 overall_return_code = return_code or overall_return_code 1205 test_results_files.append(output_paths.test_results) 1206 if options.run_ref_build: 1207 reference_benchmark_foldername = benchmark + '.reference' 1208 reference_output_paths = OutputFilePaths( 1209 isolated_out_dir, reference_benchmark_foldername).SetUp() 1210 reference_command_generator = TelemetryCommandGenerator( 1211 benchmark, 1212 options, 1213 story_selection_config=story_selection_config, 1214 is_reference=True) 1215 print( 1216 '\n### {folder} ###'.format(folder=reference_benchmark_foldername)) 1217 # We intentionally ignore the return code and test results of the 1218 # reference build. 1219 execute_telemetry_benchmark( 1220 reference_command_generator, 1221 reference_output_paths, 1222 options.xvfb, 1223 options.ignore_benchmark_exit_code, 1224 no_output_conversion=options.no_output_conversion) 1225 if 'executables' in shard_configuration: 1226 names_and_configs = shard_configuration['executables'] 1227 for (name, configuration) in names_and_configs.items(): 1228 additional_flags = [] 1229 if 'arguments' in configuration: 1230 additional_flags = configuration['arguments'] 1231 command_generator = GtestCommandGenerator( 1232 options, 1233 override_executable=configuration['path'], 1234 additional_flags=additional_flags, 1235 ignore_shard_env_vars=True) 1236 for run_num in range(options.benchmark_max_runs): 1237 output_paths = OutputFilePaths(isolated_out_dir, name).SetUp() 1238 print('\n### {folder} (attempt #{num}) ###'.format(folder=name, 1239 num=run_num)) 1240 return_code = execute_gtest_perf_test(command_generator, output_paths, 1241 options.xvfb) 1242 if return_code == 0: 1243 break 1244 overall_return_code = return_code or overall_return_code 1245 test_results_files.append(output_paths.test_results) 1246 if 'crossbench' in shard_configuration: 1247 benchmarks = shard_configuration['crossbench'] 1248 # Overwriting the "run_benchmark" with the Crossbench tool. 1249 options.executable = str(CROSSBENCH_TOOL) 1250 original_passthrough_args = options.passthrough_args.copy() 1251 for benchmark, benchmark_config in benchmarks.items(): 1252 display_name = benchmark_config.get('display_name', benchmark) 1253 if benchmark_args := benchmark_config.get('arguments', []): 1254 options.passthrough_args.extend(benchmark_args) 1255 options.benchmarks = benchmark 1256 crossbench_test = CrossbenchTest(options, isolated_out_dir) 1257 for run_num in range(options.benchmark_max_runs): 1258 print(f'\n### {display_name} (attempt #{run_num}) ###') 1259 return_code = crossbench_test.execute_benchmark( 1260 benchmark, display_name, options.passthrough_args) 1261 if return_code == 0: 1262 break 1263 overall_return_code = return_code or overall_return_code 1264 test_results_files.append( 1265 OutputFilePaths(isolated_out_dir, display_name).test_results) 1266 options.passthrough_args = original_passthrough_args.copy() 1267 1268 return overall_return_code 1269 1270 1271# This is not really a "script test" so does not need to manually add 1272# any additional compile targets. 1273def main_compile_targets(args): 1274 json.dump([], args.output) 1275 1276 1277if __name__ == '__main__': 1278 # Conform minimally to the protocol defined by ScriptTest. 1279 if 'compile_targets' in sys.argv: 1280 funcs = { 1281 'run': None, 1282 'compile_targets': main_compile_targets, 1283 } 1284 sys.exit(common.run_script(sys.argv[1:], funcs)) 1285 1286 sys.exit(main(sys.argv)) 1287