1#!/usr/bin/env vpython3 2# Copyright 2017 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Runs telemetry benchmarks and gtest perf tests. 7 8If optional argument --isolated-script-test-output=[FILENAME] is passed 9to the script, json is written to that file in the format detailed in 10//docs/testing/json-test-results-format.md. 11 12If optional argument --isolated-script-test-filter=[TEST_NAMES] is passed to 13the script, it should be a double-colon-separated ("::") list of test names, 14to run just that subset of tests. 15 16This script is intended to be the base command invoked by the isolate, 17followed by a subsequent Python script. It could be generalized to 18invoke an arbitrary executable. 19It currently runs several benchmarks. The benchmarks it will execute are 20based on the shard it is running on and the sharding_map_path. 21 22If this is executed with a gtest perf test, the flag --non-telemetry 23has to be passed in to the script so the script knows it is running 24an executable and not the run_benchmark command. 25 26This script merges test results from all the benchmarks into the one 27output.json file. The test results and perf results are also put in separate 28directories per benchmark. Two files will be present in each directory; 29perf_results.json, which is the perf specific results (with unenforced format, 30could be histogram or graph json), and test_results.json. 31 32TESTING: 33To test changes to this script, please run 34cd tools/perf 35./run_tests ScriptsSmokeTest.testRunPerformanceTests 36""" 37 38from __future__ import print_function 39 40import argparse 41import json 42import os 43import requests 44import shutil 45import sys 46import time 47import tempfile 48import traceback 49import six 50 51from collections import OrderedDict 52 53CHROMIUM_SRC_DIR = os.path.abspath( 54 os.path.join(os.path.dirname(__file__), 55 os.path.pardir, os.path.pardir)) 56 57PERF_DIR = os.path.join(CHROMIUM_SRC_DIR, 'tools', 'perf') 58sys.path.append(PERF_DIR) 59import generate_legacy_perf_dashboard_json 60from core import path_util 61 62PERF_CORE_DIR = os.path.join(PERF_DIR, 'core') 63sys.path.append(PERF_CORE_DIR) 64import results_merger 65 66# Add src/testing/ into sys.path for importing xvfb, test_env, and common. 67sys.path.append( 68 os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) 69import xvfb 70import test_env 71from scripts import common 72 73SHARD_MAPS_DIRECTORY = os.path.abspath( 74 os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, 75 'tools', 'perf', 'core', 'shard_maps')) 76 77# See https://crbug.com/923564. 78# We want to switch over to using histograms for everything, but converting from 79# the format output by gtest perf tests to histograms has introduced several 80# problems. So, only perform the conversion on tests that are whitelisted and 81# are okay with potentially encountering issues. 82GTEST_CONVERSION_WHITELIST = [ 83 'angle_perftests', 84 'base_perftests', 85 'blink_heap_perftests', 86 'blink_platform_perftests', 87 'cc_perftests', 88 'components_perftests', 89 'command_buffer_perftests', 90 'dawn_perf_tests', 91 'gpu_perftests', 92 'load_library_perf_tests', 93 'net_perftests', 94 'browser_tests', 95 'services_perftests', 96 # TODO(jmadill): Remove once migrated. http://anglebug.com/5124 97 'standalone_angle_perftests', 98 'sync_performance_tests', 99 'tracing_perftests', 100 'views_perftests', 101 'viz_perftests', 102 'wayland_client_perftests', 103 'xr.vr.common_perftests', 104] 105 106# pylint: disable=useless-object-inheritance 107 108 109class OutputFilePaths(object): 110 """Provide paths to where results outputs should be written. 111 112 The process_perf_results.py merge script later will pull all of these 113 together, so that's why they aren't in the standard locations. Also, 114 note that because of the OBBS (One Build Bot Step), Telemetry 115 has multiple tests running on a single shard, so we need to prefix 116 these locations with a directory named by the benchmark name. 117 """ 118 119 def __init__(self, isolated_out_dir, perf_test_name): 120 self.name = perf_test_name 121 self.benchmark_path = os.path.join(isolated_out_dir, perf_test_name) 122 123 def SetUp(self): 124 os.makedirs(self.benchmark_path) 125 return self 126 127 @property 128 def perf_results(self): 129 return os.path.join(self.benchmark_path, 'perf_results.json') 130 131 @property 132 def test_results(self): 133 return os.path.join(self.benchmark_path, 'test_results.json') 134 135 @property 136 def logs(self): 137 return os.path.join(self.benchmark_path, 'benchmark_log.txt') 138 139 @property 140 def csv_perf_results(self): 141 """Path for csv perf results. 142 143 Note that the chrome.perf waterfall uses the json histogram perf results 144 exclusively. csv_perf_results are implemented here in case a user script 145 passes --output-format=csv. 146 """ 147 return os.path.join(self.benchmark_path, 'perf_results.csv') 148 149 150def print_duration(step, start): 151 print('Duration of %s: %d seconds' % (step, time.time() - start)) 152 153 154def IsWindows(): 155 return sys.platform == 'cygwin' or sys.platform.startswith('win') 156 157 158class GtestCommandGenerator(object): 159 def __init__(self, options, override_executable=None, additional_flags=None, 160 ignore_shard_env_vars=False): 161 self._options = options 162 self._override_executable = override_executable 163 self._additional_flags = additional_flags or [] 164 self._ignore_shard_env_vars = ignore_shard_env_vars 165 166 def generate(self, output_dir): 167 """Generate the command to run to start the gtest perf test. 168 169 Returns: 170 list of strings, the executable and its arguments. 171 """ 172 return ([self._get_executable()] + 173 self._generate_filter_args() + 174 self._generate_repeat_args() + 175 self._generate_also_run_disabled_tests_args() + 176 self._generate_output_args(output_dir) + 177 self._generate_shard_args() + 178 self._get_additional_flags() 179 ) 180 181 @property 182 def executable_name(self): 183 """Gets the platform-independent name of the executable.""" 184 return self._override_executable or self._options.executable 185 186 def _get_executable(self): 187 executable = str(self.executable_name) 188 if IsWindows(): 189 return r'.\%s.exe' % executable 190 return './%s' % executable 191 192 def _get_additional_flags(self): 193 return self._additional_flags 194 195 def _generate_shard_args(self): 196 """Teach the gtest to ignore the environment variables. 197 198 GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS will confuse the gtest 199 and convince it to only run some of its tests. Instead run all 200 of them. 201 """ 202 if self._ignore_shard_env_vars: 203 return ['--test-launcher-total-shards=1', '--test-launcher-shard-index=0'] 204 return [] 205 206 def _generate_filter_args(self): 207 if self._options.isolated_script_test_filter: 208 filter_list = common.extract_filter_list( 209 self._options.isolated_script_test_filter) 210 return ['--gtest_filter=' + ':'.join(filter_list)] 211 return [] 212 213 def _generate_repeat_args(self): 214 # TODO(crbug.com/920002): Support --isolated-script-test-repeat. 215 return [] 216 217 def _generate_also_run_disabled_tests_args(self): 218 # TODO(crbug.com/920002): Support 219 # --isolated-script-test-also-run-disabled-tests. 220 return [] 221 222 def _generate_output_args(self, output_dir): 223 output_args = [] 224 if self._options.use_gtest_benchmark_script: 225 output_args.append('--output-dir=' + output_dir) 226 # These flags are to make sure that test output perf metrics in the log. 227 if not '--verbose' in self._get_additional_flags(): 228 output_args.append('--verbose') 229 if (not '--test-launcher-print-test-stdio=always' 230 in self._get_additional_flags()): 231 output_args.append('--test-launcher-print-test-stdio=always') 232 return output_args 233 234 235def write_simple_test_results(return_code, output_filepath, benchmark_name): 236 # TODO(crbug.com/1115658): Fix to output 237 # https://chromium.googlesource.com/chromium/src/+/main/docs/testing/json_test_results_format.md 238 # for each test rather than this summary. 239 # Append the shard index to the end of the name so that the merge script 240 # doesn't blow up trying to merge unmergeable results. 241 benchmark_name += '_shard_%s' % os.environ.get('GTEST_SHARD_INDEX', '0') 242 output_json = { 243 'tests': { 244 benchmark_name: { 245 'expected': 'PASS', 246 'actual': 'FAIL' if return_code else 'PASS', 247 'is_unexpected': bool(return_code), 248 }, 249 }, 250 'interrupted': False, 251 'path_delimiter': '/', 252 'version': 3, 253 'seconds_since_epoch': time.time(), 254 'num_failures_by_type': { 255 'FAIL': 1 if return_code else 0, 256 'PASS': 0 if return_code else 1, 257 }, 258 } 259 with open(output_filepath, 'w') as fh: 260 json.dump(output_json, fh) 261 262 263def upload_simple_test_results(return_code, benchmark_name): 264 # TODO(crbug.com/1115658): Fix to upload results for each test rather than 265 # this summary. 266 try: 267 with open(os.environ['LUCI_CONTEXT']) as f: 268 sink = json.load(f)['result_sink'] 269 except KeyError: 270 return 271 272 if return_code: 273 summary = '<p>Benchmark failed with status code %d</p>' % return_code 274 else: 275 summary = '<p>Benchmark passed</p>' 276 277 result_json = { 278 'testResults': [{ 279 'testId': benchmark_name, 280 'expected': not return_code, 281 'status': 'FAIL' if return_code else 'PASS', 282 'summaryHtml': summary, 283 'tags': [{'key': 'exit_code', 'value': str(return_code)}], 284 }] 285 } 286 287 res = requests.post( 288 url='http://%s/prpc/luci.resultsink.v1.Sink/ReportTestResults' % 289 sink['address'], 290 headers={ 291 'Content-Type': 'application/json', 292 'Accept': 'application/json', 293 'Authorization': 'ResultSink %s' % sink['auth_token'], 294 }, 295 data=json.dumps(result_json)) 296 res.raise_for_status() 297 298 299def execute_gtest_perf_test(command_generator, output_paths, use_xvfb=False, 300 is_unittest=False): 301 start = time.time() 302 303 env = os.environ.copy() 304 env['CHROME_HEADLESS'] = '1' 305 #TODO(crbug/1138988): Some gtests do not implements the unit_test_launcher.cc. 306 # As a result, they will not respect the arguments added by 307 # _generate_shard_args() and will still use the values of GTEST_SHARD_INDEX 308 # and GTEST_TOTAL_SHARDS to run part of the tests. 309 # Removing those environment variables as a workaround. 310 if command_generator._ignore_shard_env_vars: 311 if 'GTEST_TOTAL_SHARDS' in env: 312 env.pop('GTEST_TOTAL_SHARDS') 313 if 'GTEST_SHARD_INDEX' in env: 314 env.pop('GTEST_SHARD_INDEX') 315 316 return_code = 0 317 try: 318 command = command_generator.generate(output_paths.benchmark_path) 319 if use_xvfb: 320 # When running with xvfb, we currently output both to stdout and to the 321 # file. It would be better to only output to the file to keep the logs 322 # clean. 323 return_code = xvfb.run_executable( 324 command, env, stdoutfile=output_paths.logs) 325 else: 326 with open(output_paths.logs, 'w') as handle: 327 try: 328 return_code = test_env.run_command_output_to_handle( 329 command, handle, env=env) 330 except OSError as e: 331 print('Command to run gtest perf test %s failed with an OSError: %s' % 332 (output_paths.name, e)) 333 return_code = 1 334 if (not os.path.exists(output_paths.perf_results) and 335 os.path.exists(output_paths.logs)): 336 # Get the correct json format from the stdout to write to the perf 337 # results file if gtest does not generate one. 338 results_processor = generate_legacy_perf_dashboard_json.\ 339 LegacyResultsProcessor() 340 graph_json_string = results_processor.GenerateJsonResults( 341 output_paths.logs) 342 with open(output_paths.perf_results, 'w') as fh: 343 fh.write(graph_json_string) 344 except Exception: 345 traceback.print_exc() 346 return_code = 1 347 if os.path.exists(output_paths.perf_results): 348 executable_name = command_generator.executable_name 349 if executable_name.startswith('bin/run_'): 350 # The executable is a wrapper used by Fuchsia. Remove the prefix to get 351 # the actual executable name. 352 executable_name = executable_name[8:] 353 if executable_name in GTEST_CONVERSION_WHITELIST: 354 with path_util.SysPath(path_util.GetTracingDir()): 355 # pylint: disable=no-name-in-module,import-outside-toplevel 356 from tracing.value import gtest_json_converter 357 # pylint: enable=no-name-in-module,import-outside-toplevel 358 gtest_json_converter.ConvertGtestJsonFile(output_paths.perf_results) 359 else: 360 print('ERROR: gtest perf test %s did not generate perf output' % 361 output_paths.name) 362 return_code = 1 363 write_simple_test_results(return_code, output_paths.test_results, 364 output_paths.name) 365 if not is_unittest: 366 upload_simple_test_results(return_code, output_paths.name) 367 368 print_duration( 369 'executing gtest %s' % command_generator.executable_name, start) 370 371 return return_code 372 373 374class _TelemetryFilterArgument(object): 375 def __init__(self, filter_string): 376 self.benchmark, self.story = filter_string.split('/') 377 378 379class TelemetryCommandGenerator(object): 380 def __init__(self, benchmark, options, 381 story_selection_config=None, is_reference=False): 382 self.benchmark = benchmark 383 self._options = options 384 self._story_selection_config = story_selection_config 385 self._is_reference = is_reference 386 387 def generate(self, output_dir): 388 """Generate the command to run to start the benchmark. 389 390 Args: 391 output_dir: The directory to configure the command to put output files 392 into. 393 394 Returns: 395 list of strings, the executable and its arguments. 396 """ 397 return ([sys.executable] + self._options.executable.split(' ') + 398 [self.benchmark] + 399 self._generate_filter_args() + 400 self._generate_also_run_disabled_tests_args() + 401 self._generate_output_args(output_dir) + 402 self._generate_story_selection_args() + 403 # passthrough args must be before reference args and repeat args: 404 # crbug.com/928928, crbug.com/894254#c78 405 self._get_passthrough_args() + 406 self._generate_syslog_args() + 407 self._generate_repeat_args() + 408 self._generate_reference_build_args() 409 ) 410 411 def _get_passthrough_args(self): 412 return self._options.passthrough_args 413 414 def _generate_filter_args(self): 415 if self._options.isolated_script_test_filter: 416 filter_list = common.extract_filter_list( 417 self._options.isolated_script_test_filter) 418 filter_arguments = [_TelemetryFilterArgument(f) for f in filter_list] 419 applicable_stories = [ 420 f.story for f in filter_arguments if f.benchmark == self.benchmark] 421 # Need to convert this to a valid regex. 422 filter_regex = '(' + '|'.join(applicable_stories) + ')' 423 return ['--story-filter=' + filter_regex] 424 return [] 425 426 def _generate_repeat_args(self): 427 if self._options.isolated_script_test_repeat: 428 return ['--pageset-repeat=' + str( 429 self._options.isolated_script_test_repeat)] 430 return [] 431 432 def _generate_also_run_disabled_tests_args(self): 433 if self._options.isolated_script_test_also_run_disabled_tests: 434 return ['--also-run-disabled-tests'] 435 return [] 436 437 def _generate_output_args(self, output_dir): 438 return ['--output-format=json-test-results', 439 '--output-format=histograms', 440 '--output-dir=' + output_dir] 441 442 def _generate_story_selection_args(self): 443 """Returns arguments that limit the stories to be run inside the benchmark. 444 """ 445 selection_args = [] 446 if self._story_selection_config: 447 if 'begin' in self._story_selection_config: 448 selection_args.append('--story-shard-begin-index=%d' % ( 449 self._story_selection_config['begin'])) 450 if 'end' in self._story_selection_config: 451 selection_args.append('--story-shard-end-index=%d' % ( 452 self._story_selection_config['end'])) 453 if 'sections' in self._story_selection_config: 454 range_string = self._generate_story_index_ranges( 455 self._story_selection_config['sections']) 456 if range_string: 457 selection_args.append('--story-shard-indexes=%s' % range_string) 458 if self._story_selection_config.get('abridged', True): 459 selection_args.append('--run-abridged-story-set') 460 return selection_args 461 462 def _generate_syslog_args(self): 463 if self._options.per_test_logs_dir: 464 isolated_out_dir = os.path.dirname( 465 self._options.isolated_script_test_output) 466 return ['--logs-dir', os.path.join( 467 isolated_out_dir, 468 self.benchmark)] 469 return [] 470 471 472 def _generate_story_index_ranges(self, sections): 473 range_string = '' 474 for section in sections: 475 begin = section.get('begin', '') 476 end = section.get('end', '') 477 # If there only one story in the range, we only keep its index. 478 # In general, we expect either begin or end, or both. 479 if begin != '' and end != '' and end - begin == 1: 480 new_range = str(begin) 481 elif begin != '' or end != '': 482 new_range = '%s-%s' % (str(begin), str(end)) 483 else: 484 raise ValueError('Index ranges in "sections" in shard map should have' 485 'at least one of "begin" and "end": %s' % str(section)) 486 if range_string: 487 range_string += ',%s' % new_range 488 else: 489 range_string = new_range 490 return range_string 491 492 493 def _generate_reference_build_args(self): 494 if self._is_reference: 495 reference_browser_flag = '--browser=reference' 496 # TODO(crbug.com/1038137): Make the logic generic once more reference 497 # settings are added 498 if '--browser=android-chrome-bundle' in self._get_passthrough_args(): 499 reference_browser_flag = '--browser=reference-android-chrome-bundle' 500 return [reference_browser_flag, 501 '--max-failures=5'] 502 return [] 503 504 505def execute_telemetry_benchmark( 506 command_generator, output_paths, use_xvfb=False, 507 return_exit_code_zero=False): 508 start = time.time() 509 510 env = os.environ.copy() 511 env['CHROME_HEADLESS'] = '1' 512 513 return_code = 1 514 temp_dir = tempfile.mkdtemp('telemetry') 515 infra_failure = False 516 try: 517 command = command_generator.generate(temp_dir) 518 if use_xvfb: 519 # When running with xvfb, we currently output both to stdout and to the 520 # file. It would be better to only output to the file to keep the logs 521 # clean. 522 return_code = xvfb.run_executable( 523 command, env=env, stdoutfile=output_paths.logs) 524 else: 525 with open(output_paths.logs, 'w') as handle: 526 return_code = test_env.run_command_output_to_handle( 527 command, handle, env=env) 528 expected_results_filename = os.path.join(temp_dir, 'test-results.json') 529 if os.path.exists(expected_results_filename): 530 shutil.move(expected_results_filename, output_paths.test_results) 531 else: 532 common.write_interrupted_test_results_to(output_paths.test_results, start) 533 expected_perf_filename = os.path.join(temp_dir, 'histograms.json') 534 shutil.move(expected_perf_filename, output_paths.perf_results) 535 536 csv_file_path = os.path.join(temp_dir, 'results.csv') 537 if os.path.isfile(csv_file_path): 538 shutil.move(csv_file_path, output_paths.csv_perf_results) 539 except Exception: 540 print('The following exception may have prevented the code from ' 541 'outputing structured test results and perf results output:') 542 print(traceback.format_exc()) 543 infra_failure = True 544 finally: 545 # On swarming bots, don't remove output directory, since Result Sink might 546 # still be uploading files to Result DB. Also, swarming bots automatically 547 # clean up at the end of each task. 548 if 'SWARMING_TASK_ID' not in os.environ: 549 # Add ignore_errors=True because otherwise rmtree may fail due to leaky 550 # processes of tests are still holding opened handles to files under 551 # |tempfile_dir|. For example, see crbug.com/865896 552 shutil.rmtree(temp_dir, ignore_errors=True) 553 554 print_duration('executing benchmark %s' % command_generator.benchmark, start) 555 556 if infra_failure: 557 print('There was an infrastructure error encountered during the run. ' 558 'Please check the logs above for details') 559 return 1 560 561 # Telemetry sets exit code to -1 to indicate that no stories were run. This 562 # becomes 255 on linux because linux doesn't support -1 so it does modulo: 563 # -1 % 256 == 255. 564 # TODO(crbug.com/1019139): Make 111 be the exit code that means 565 # "no stories were run.". 566 if return_code in (111, -1, 255): 567 print('Exit code %s indicates that no stories were run, so we are marking ' 568 'this as a success.' % return_code) 569 return 0 570 if return_code: 571 if return_exit_code_zero: 572 print ('run_benchmark returned exit code ' + str(return_code) 573 + ' which indicates there were test failures in the run.') 574 return 0 575 return return_code 576 return 0 577 578def parse_arguments(args): 579 parser = argparse.ArgumentParser() 580 parser.add_argument('executable', help='The name of the executable to run.') 581 parser.add_argument( 582 '--isolated-script-test-output', required=True) 583 # The following two flags may be passed in sometimes by Pinpoint 584 # or by the recipe, but they don't do anything. crbug.com/927482. 585 parser.add_argument( 586 '--isolated-script-test-chartjson-output', required=False) 587 parser.add_argument( 588 '--isolated-script-test-perf-output', required=False) 589 590 parser.add_argument( 591 '--isolated-script-test-filter', type=str, required=False) 592 593 # Note that the following three arguments are only supported by Telemetry 594 # tests right now. See crbug.com/920002. 595 parser.add_argument( 596 '--isolated-script-test-repeat', type=int, required=False) 597 parser.add_argument( 598 '--isolated-script-test-launcher-retry-limit', type=int, required=False, 599 choices=[0]) # Telemetry does not support retries. crbug.com/894254#c21 600 parser.add_argument( 601 '--isolated-script-test-also-run-disabled-tests', 602 default=False, action='store_true', required=False) 603 parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') 604 parser.add_argument('--non-telemetry', 605 help='Type of perf test', type=bool, default=False) 606 parser.add_argument('--gtest-benchmark-name', 607 help='Name of the gtest benchmark', type=str, 608 required=False) 609 parser.add_argument('--use-gtest-benchmark-script', 610 help='Whether gtest is invoked via benchmark script.', 611 default=False, action='store_true') 612 613 parser.add_argument('--benchmarks', 614 help='Comma separated list of benchmark names' 615 ' to run in lieu of indexing into our benchmark bot maps', 616 required=False) 617 # crbug.com/1236245: This allows for per-benchmark device logs. 618 parser.add_argument('--per-test-logs-dir', 619 help='Require --logs-dir args for test', required=False, 620 default=False, action='store_true') 621 # Some executions may have a different sharding scheme and/or set of tests. 622 # These files must live in src/tools/perf/core/shard_maps 623 parser.add_argument('--test-shard-map-filename', type=str, required=False) 624 parser.add_argument('--run-ref-build', 625 help='Run test on reference browser', action='store_true') 626 parser.add_argument('--passthrough-arg', 627 help='Arguments to pass directly through to the test ' 628 'executable.', action='append', 629 dest='passthrough_args', 630 default=[]) 631 parser.add_argument('--use-dynamic-shards', 632 help='If set, use dynamic shardmap instead of the file.', 633 action='store_true', 634 required=False 635 ) 636 parser.add_argument('--dynamic-shardmap', 637 help='The dynamically generated shardmap string used to ' 638 'replace the static shardmap file.', 639 type=str, 640 required=False) 641 parser.add_argument('--ignore-benchmark-exit-code', 642 help='If set, return an exit code 0 even if there' 643 + ' are benchmark failures', 644 action='store_true', 645 required=False 646 ) 647 options, leftover_args = parser.parse_known_args(args) 648 options.passthrough_args.extend(leftover_args) 649 return options 650 651 652def main(sys_args): 653 args = sys_args[1:] # Skip program name. 654 options = parse_arguments(args) 655 isolated_out_dir = os.path.dirname(options.isolated_script_test_output) 656 overall_return_code = 0 657 # This is a list of test results files to be merged into a standard 658 # output.json file for use by infrastructure including FindIt. 659 # This list should not contain reference build runs 660 # since we do not monitor those. Also, merging test reference build results 661 # with standard build results may not work properly. 662 test_results_files = [] 663 664 print('Running a series of performance test subprocesses. Logs, performance\n' 665 'results, and test results JSON will be saved in a subfolder of the\n' 666 'isolated output directory. Inside the hash marks in the following\n' 667 'lines is the name of the subfolder to find results in.\n') 668 669 if options.non_telemetry: 670 benchmark_name = options.gtest_benchmark_name 671 passthrough_args = options.passthrough_args 672 # crbug/1146949#c15 673 # In the case that pinpoint passes all arguments to swarming through http 674 # request, the passthrough_args are converted into a comma-separated string. 675 if passthrough_args and isinstance(passthrough_args, six.text_type): 676 passthrough_args = passthrough_args.split(',') 677 # With --non-telemetry, the gtest executable file path will be passed in as 678 # options.executable, which is different from running on shard map. Thus, 679 # we don't override executable as we do in running on shard map. 680 command_generator = GtestCommandGenerator( 681 options, additional_flags=passthrough_args, ignore_shard_env_vars=True) 682 # Fallback to use the name of the executable if flag isn't set. 683 # TODO(crbug.com/870899): remove fallback logic and raise parser error if 684 # --non-telemetry is set but --gtest-benchmark-name is not set once pinpoint 685 # is converted to always pass --gtest-benchmark-name flag. 686 if not benchmark_name: 687 benchmark_name = options.executable 688 output_paths = OutputFilePaths(isolated_out_dir, benchmark_name).SetUp() 689 print('\n### {folder} ###'.format(folder=benchmark_name)) 690 overall_return_code = execute_gtest_perf_test( 691 command_generator, output_paths, options.xvfb) 692 test_results_files.append(output_paths.test_results) 693 else: 694 if options.use_dynamic_shards: 695 shard_map_str = options.dynamic_shardmap 696 shard_map = json.loads(shard_map_str, object_pairs_hook=OrderedDict) 697 shard_map_path = os.path.join(SHARD_MAPS_DIRECTORY, 698 options.test_shard_map_filename) 699 with open(shard_map_path, 'w') as f: 700 json.dump(shard_map, f, indent=4, separators=(',', ': ')) 701 shutil.copyfile( 702 shard_map_path, 703 os.path.join(isolated_out_dir, 'benchmarks_shard_map.json')) 704 overall_return_code = _run_benchmarks_on_shardmap( 705 shard_map, options, isolated_out_dir, test_results_files 706 ) 707 # If the user has supplied a list of benchmark names, execute those instead 708 # of using the shard map. 709 elif options.benchmarks: 710 benchmarks = options.benchmarks.split(',') 711 for benchmark in benchmarks: 712 output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp() 713 command_generator = TelemetryCommandGenerator( 714 benchmark, options) 715 print('\n### {folder} ###'.format(folder=benchmark)) 716 return_code = execute_telemetry_benchmark( 717 command_generator, output_paths, options.xvfb, 718 options.ignore_benchmark_exit_code) 719 overall_return_code = return_code or overall_return_code 720 test_results_files.append(output_paths.test_results) 721 if options.run_ref_build: 722 print('Not running reference build. --run-ref-build argument is only ' 723 'supported for sharded benchmarks. It is simple to support ' 724 'this for unsharded --benchmarks if needed.') 725 elif options.test_shard_map_filename: 726 # First determine what shard we are running on to know how to 727 # index into the bot map to get list of telemetry benchmarks to run. 728 shard_map_path = os.path.join(SHARD_MAPS_DIRECTORY, 729 options.test_shard_map_filename) 730 # Copy sharding map file to isolated_out_dir so that the merge script 731 # can collect it later. 732 shutil.copyfile( 733 shard_map_path, 734 os.path.join(isolated_out_dir, 'benchmarks_shard_map.json')) 735 with open(shard_map_path) as f: 736 shard_map = json.load(f) 737 overall_return_code = _run_benchmarks_on_shardmap( 738 shard_map, options, isolated_out_dir, test_results_files 739 ) 740 else: 741 raise Exception('Telemetry tests must provide either a shard map or a ' 742 '--benchmarks list so that we know which stories to run.') 743 744 test_results_list = [] 745 for test_results_file in test_results_files: 746 if os.path.exists(test_results_file): 747 with open(test_results_file, 'r') as fh: 748 test_results_list.append(json.load(fh)) 749 merged_test_results = results_merger.merge_test_results(test_results_list) 750 with open(options.isolated_script_test_output, 'w') as f: 751 json.dump(merged_test_results, f) 752 753 return overall_return_code 754 755def _run_benchmarks_on_shardmap( 756 shard_map, options, isolated_out_dir, test_results_files): 757 overall_return_code = 0 758 shard_index = None 759 env = os.environ.copy() 760 if 'GTEST_SHARD_INDEX' in env: 761 shard_index = env['GTEST_SHARD_INDEX'] 762 # TODO(crbug.com/972844): shard environment variables are not specified 763 # for single-shard shard runs. 764 if not shard_index: 765 shard_map_has_multiple_shards = bool(shard_map.get('1', False)) 766 if not shard_map_has_multiple_shards: 767 shard_index = '0' 768 if not shard_index: 769 raise Exception( 770 'Sharded Telemetry perf tests must either specify --benchmarks ' 771 'list or have GTEST_SHARD_INDEX environment variable present.') 772 shard_configuration = shard_map[shard_index] 773 assert ('benchmarks' in shard_configuration or 774 'executables' in shard_configuration), ( 775 'Every shard must have benchmarks or executables associated ' 776 'with it.') 777 if 'benchmarks' in shard_configuration: 778 benchmarks_and_configs = shard_configuration['benchmarks'] 779 for (benchmark, story_selection_config) in benchmarks_and_configs.items(): 780 # Need to run the benchmark on both latest browser and reference 781 # build. 782 output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp() 783 command_generator = TelemetryCommandGenerator( 784 benchmark, options, 785 story_selection_config=story_selection_config) 786 print('\n### {folder} ###'.format(folder=benchmark)) 787 return_code = execute_telemetry_benchmark( 788 command_generator, output_paths, options.xvfb, 789 options.ignore_benchmark_exit_code) 790 overall_return_code = return_code or overall_return_code 791 test_results_files.append(output_paths.test_results) 792 if options.run_ref_build: 793 reference_benchmark_foldername = benchmark + '.reference' 794 reference_output_paths = OutputFilePaths( 795 isolated_out_dir, reference_benchmark_foldername).SetUp() 796 reference_command_generator = TelemetryCommandGenerator( 797 benchmark, options, 798 story_selection_config=story_selection_config, 799 is_reference=True) 800 print('\n### {folder} ###'.format( 801 folder=reference_benchmark_foldername)) 802 # We intentionally ignore the return code and test results of the 803 # reference build. 804 execute_telemetry_benchmark( 805 reference_command_generator, reference_output_paths, 806 options.xvfb, options.ignore_benchmark_exit_code) 807 if 'executables' in shard_configuration: 808 names_and_configs = shard_configuration['executables'] 809 for (name, configuration) in names_and_configs.items(): 810 additional_flags = [] 811 if 'arguments' in configuration: 812 additional_flags = configuration['arguments'] 813 command_generator = GtestCommandGenerator( 814 options, override_executable=configuration['path'], 815 additional_flags=additional_flags, ignore_shard_env_vars=True) 816 output_paths = OutputFilePaths(isolated_out_dir, name).SetUp() 817 print('\n### {folder} ###'.format(folder=name)) 818 return_code = execute_gtest_perf_test( 819 command_generator, output_paths, options.xvfb) 820 overall_return_code = return_code or overall_return_code 821 test_results_files.append(output_paths.test_results) 822 823 return overall_return_code 824 825# This is not really a "script test" so does not need to manually add 826# any additional compile targets. 827def main_compile_targets(args): 828 json.dump([], args.output) 829 830 831if __name__ == '__main__': 832 # Conform minimally to the protocol defined by ScriptTest. 833 if 'compile_targets' in sys.argv: 834 funcs = { 835 'run': None, 836 'compile_targets': main_compile_targets, 837 } 838 sys.exit(common.run_script(sys.argv[1:], funcs)) 839 840 sys.exit(main(sys.argv)) 841