1#! /usr/bin/env vpython3 2# 3# Copyright 2020 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7# restricted_trace_gold_tests.py: 8# Uses Skia Gold (https://skia.org/dev/testing/skiagold) to run pixel tests with ANGLE traces. 9# 10# Requires vpython to run standalone. Run with --help for usage instructions. 11 12import argparse 13import contextlib 14import json 15import logging 16import os 17import pathlib 18import platform 19import re 20import shutil 21import sys 22import tempfile 23import time 24import traceback 25 26 27PY_UTILS = str(pathlib.Path(__file__).resolve().parents[1] / 'py_utils') 28if PY_UTILS not in sys.path: 29 os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS) 30import android_helper 31import angle_path_util 32import angle_test_util 33from skia_gold import angle_skia_gold_properties 34from skia_gold import angle_skia_gold_session_manager 35 36angle_path_util.AddDepsDirToPath('testing/scripts') 37import common 38 39 40DEFAULT_TEST_SUITE = angle_test_util.ANGLE_TRACE_TEST_SUITE 41DEFAULT_TEST_PREFIX = 'TraceTest.' 42DEFAULT_SCREENSHOT_PREFIX = 'angle_vulkan_' 43SWIFTSHADER_SCREENSHOT_PREFIX = 'angle_vulkan_swiftshader_' 44DEFAULT_BATCH_SIZE = 5 45DEFAULT_LOG = 'info' 46DEFAULT_GOLD_INSTANCE = 'angle' 47 48# Test expectations 49FAIL = 'FAIL' 50PASS = 'PASS' 51SKIP = 'SKIP' 52 53 54@contextlib.contextmanager 55def temporary_dir(prefix=''): 56 path = tempfile.mkdtemp(prefix=prefix) 57 try: 58 yield path 59 finally: 60 logging.info("Removing temporary directory: %s" % path) 61 shutil.rmtree(path) 62 63 64def add_skia_gold_args(parser): 65 group = parser.add_argument_group('Skia Gold Arguments') 66 group.add_argument('--git-revision', help='Revision being tested.', default=None) 67 group.add_argument( 68 '--gerrit-issue', help='For Skia Gold integration. Gerrit issue ID.', default='') 69 group.add_argument( 70 '--gerrit-patchset', 71 help='For Skia Gold integration. Gerrit patch set number.', 72 default='') 73 group.add_argument( 74 '--buildbucket-id', help='For Skia Gold integration. Buildbucket build ID.', default='') 75 group.add_argument( 76 '--bypass-skia-gold-functionality', 77 action='store_true', 78 default=False, 79 help='Bypass all interaction with Skia Gold, effectively disabling the ' 80 'image comparison portion of any tests that use Gold. Only meant to ' 81 'be used in case a Gold outage occurs and cannot be fixed quickly.') 82 local_group = group.add_mutually_exclusive_group() 83 local_group.add_argument( 84 '--local-pixel-tests', 85 action='store_true', 86 default=None, 87 help='Specifies to run the test harness in local run mode or not. When ' 88 'run in local mode, uploading to Gold is disabled and links to ' 89 'help with local debugging are output. Running in local mode also ' 90 'implies --no-luci-auth. If both this and --no-local-pixel-tests are ' 91 'left unset, the test harness will attempt to detect whether it is ' 92 'running on a workstation or not and set this option accordingly.') 93 local_group.add_argument( 94 '--no-local-pixel-tests', 95 action='store_false', 96 dest='local_pixel_tests', 97 help='Specifies to run the test harness in non-local (bot) mode. When ' 98 'run in this mode, data is actually uploaded to Gold and triage links ' 99 'arge generated. If both this and --local-pixel-tests are left unset, ' 100 'the test harness will attempt to detect whether it is running on a ' 101 'workstation or not and set this option accordingly.') 102 group.add_argument( 103 '--no-luci-auth', 104 action='store_true', 105 default=False, 106 help='Don\'t use the service account provided by LUCI for ' 107 'authentication for Skia Gold, instead relying on gsutil to be ' 108 'pre-authenticated. Meant for testing locally instead of on the bots.') 109 110 111def run_angle_system_info_test(sysinfo_args, args, env): 112 with temporary_dir() as temp_dir: 113 sysinfo_args += ['--render-test-output-dir=' + temp_dir] 114 115 result, _, _ = angle_test_util.RunTestSuite( 116 'angle_system_info_test', sysinfo_args, env, use_xvfb=args.xvfb) 117 if result != 0: 118 raise Exception('Error getting system info.') 119 120 with open(os.path.join(temp_dir, 'angle_system_info.json')) as f: 121 return json.load(f) 122 123 124def to_hex(num): 125 return hex(int(num)) 126 127 128def to_hex_or_none(num): 129 return 'None' if num == None else to_hex(num) 130 131 132def to_non_empty_string_or_none(val): 133 return 'None' if val == '' else str(val) 134 135 136def to_non_empty_string_or_none_dict(d, key): 137 return 'None' if not key in d else to_non_empty_string_or_none(d[key]) 138 139 140def get_skia_gold_keys(args, env): 141 """Get all the JSON metadata that will be passed to golctl.""" 142 # All values need to be strings, otherwise goldctl fails. 143 144 # Only call this method one time 145 if hasattr(get_skia_gold_keys, 'called') and get_skia_gold_keys.called: 146 logging.exception('get_skia_gold_keys may only be called once') 147 get_skia_gold_keys.called = True 148 149 sysinfo_args = ['--vulkan', '-v'] 150 if args.swiftshader: 151 sysinfo_args.append('--swiftshader') 152 153 if angle_test_util.IsAndroid(): 154 json_data = android_helper.AngleSystemInfo(sysinfo_args) 155 logging.info(json_data) 156 os_name = 'Android' 157 os_version = android_helper.GetBuildFingerprint() 158 else: 159 json_data = run_angle_system_info_test(sysinfo_args, args, env) 160 os_name = to_non_empty_string_or_none(platform.system()) 161 os_version = to_non_empty_string_or_none(platform.version()) 162 163 if len(json_data.get('gpus', [])) == 0 or not 'activeGPUIndex' in json_data: 164 raise Exception('Error getting system info.') 165 166 active_gpu = json_data['gpus'][json_data['activeGPUIndex']] 167 168 angle_keys = { 169 'vendor_id': to_hex_or_none(active_gpu['vendorId']), 170 'device_id': to_hex_or_none(active_gpu['deviceId']), 171 'model_name': to_non_empty_string_or_none_dict(active_gpu, 'machineModelVersion'), 172 'manufacturer_name': to_non_empty_string_or_none_dict(active_gpu, 'machineManufacturer'), 173 'os': os_name, 174 'os_version': os_version, 175 'driver_version': to_non_empty_string_or_none_dict(active_gpu, 'driverVersion'), 176 'driver_vendor': to_non_empty_string_or_none_dict(active_gpu, 'driverVendor'), 177 } 178 179 return angle_keys 180 181 182def output_diff_local_files(gold_session, image_name): 183 """Logs the local diff image files from the given SkiaGoldSession. 184 185 Args: 186 gold_session: A skia_gold_session.SkiaGoldSession instance to pull files 187 from. 188 image_name: A string containing the name of the image/test that was 189 compared. 190 """ 191 given_file = gold_session.GetGivenImageLink(image_name) 192 closest_file = gold_session.GetClosestImageLink(image_name) 193 diff_file = gold_session.GetDiffImageLink(image_name) 194 failure_message = 'Unable to retrieve link' 195 logging.error('Generated image: %s', given_file or failure_message) 196 logging.error('Closest image: %s', closest_file or failure_message) 197 logging.error('Diff image: %s', diff_file or failure_message) 198 199 200def get_trace_key_frame(trace): 201 # read trace info 202 json_name = os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 'restricted_traces', 203 trace, trace + '.json') 204 with open(json_name) as fp: 205 trace_info = json.load(fp) 206 207 # Check its metadata for a keyframe 208 keyframe = '' 209 if 'KeyFrames' in trace_info['TraceMetadata']: 210 # KeyFrames is an array, but we only use the first value for now 211 keyframe = str(trace_info['TraceMetadata']['KeyFrames'][0]) 212 logging.info('trace %s is using a keyframe of %s' % (trace, keyframe)) 213 214 return keyframe 215 216 217def upload_test_result_to_skia_gold(args, gold_session_manager, gold_session, gold_properties, 218 screenshot_dir, trace, artifacts): 219 """Compares the given image using Skia Gold and uploads the result. 220 221 No uploading is done if the test is being run in local run mode. Compares 222 the given screenshot to baselines provided by Gold, raising an Exception if 223 a match is not found. 224 225 Args: 226 args: Command line options. 227 gold_session_manager: Skia Gold session manager. 228 gold_session: Skia Gold session. 229 gold_properties: Skia Gold properties. 230 screenshot_dir: directory where the test stores screenshots. 231 trace: base name of the trace being checked. 232 artifacts: dictionary of JSON artifacts to pass to the result merger. 233 """ 234 235 use_luci = not (gold_properties.local_pixel_tests or gold_properties.no_luci_auth) 236 237 # Determine if this trace is using a keyframe 238 image_name = trace 239 keyframe = get_trace_key_frame(trace) 240 if keyframe != '': 241 image_name = trace + '_frame' + keyframe 242 logging.debug('Using %s as image_name for upload' % image_name) 243 244 # Note: this would be better done by iterating the screenshot directory. 245 prefix = SWIFTSHADER_SCREENSHOT_PREFIX if args.swiftshader else DEFAULT_SCREENSHOT_PREFIX 246 png_file_name = os.path.join(screenshot_dir, prefix + image_name + '.png') 247 248 if not os.path.isfile(png_file_name): 249 raise Exception('Screenshot not found: ' + png_file_name) 250 251 if args.use_permissive_pixel_comparison: 252 # These arguments cause Gold to use the sample area inexact matching 253 # algorithm. It is set to allow any of up to 3 pixels in each 4x4 group 254 # of pixels to differ by any amount. Pixels that differ by a max of 1 255 # on all channels (i.e. have differences that can be attributed to 256 # rounding errors) do not count towards this limit. 257 # 258 # An image that passes due to this logic is auto-approved as a new good 259 # image. 260 inexact_matching_args = [ 261 '--add-test-optional-key', 262 'image_matching_algorithm:sample_area', 263 '--add-test-optional-key', 264 'sample_area_width:4', 265 '--add-test-optional-key', 266 'sample_area_max_different_pixels_per_area:3', 267 '--add-test-optional-key', 268 'sample_area_channel_delta_threshold:1', 269 ] 270 else: 271 # These arguments cause Gold to use the fuzzy inexact matching 272 # algorithm. It is set to allow up to 20k pixels to differ by 1 on all 273 # channels, which is meant to help reduce triage overhead caused by new 274 # images from rounding differences. 275 # 276 # The max number of pixels is fairly arbitrary, but the diff threshold 277 # is intentional since we don't want to let in any changes that can't be 278 # attributed to rounding errors. 279 # 280 # An image that passes due to this logic is auto-approved as a new good 281 # image. 282 inexact_matching_args = [ 283 '--add-test-optional-key', 284 'image_matching_algorithm:fuzzy', 285 '--add-test-optional-key', 286 'fuzzy_max_different_pixels:20000', 287 '--add-test-optional-key', 288 'fuzzy_pixel_per_channel_delta_threshold:1', 289 ] 290 291 status, error = gold_session.RunComparison( 292 name=image_name, 293 png_file=png_file_name, 294 use_luci=use_luci, 295 inexact_matching_args=inexact_matching_args) 296 297 artifact_name = os.path.basename(png_file_name) 298 artifacts[artifact_name] = [artifact_name] 299 300 if not status: 301 return PASS 302 303 status_codes = gold_session_manager.GetSessionClass().StatusCodes 304 if status == status_codes.AUTH_FAILURE: 305 logging.error('Gold authentication failed with output %s', error) 306 elif status == status_codes.INIT_FAILURE: 307 logging.error('Gold initialization failed with output %s', error) 308 elif status == status_codes.COMPARISON_FAILURE_REMOTE: 309 _, triage_link = gold_session.GetTriageLinks(image_name) 310 if not triage_link: 311 logging.error('Failed to get triage link for %s, raw output: %s', image_name, error) 312 logging.error('Reason for no triage link: %s', 313 gold_session.GetTriageLinkOmissionReason(image_name)) 314 if gold_properties.IsTryjobRun(): 315 # Pick "show all results" so we can see the tryjob images by default. 316 triage_link += '&master=true' 317 artifacts['triage_link_for_entire_cl'] = [triage_link] 318 else: 319 artifacts['gold_triage_link'] = [triage_link] 320 elif status == status_codes.COMPARISON_FAILURE_LOCAL: 321 logging.error('Local comparison failed. Local diff files:') 322 output_diff_local_files(gold_session, image_name) 323 elif status == status_codes.LOCAL_DIFF_FAILURE: 324 logging.error( 325 'Local comparison failed and an error occurred during diff ' 326 'generation: %s', error) 327 # There might be some files, so try outputting them. 328 logging.error('Local diff files:') 329 output_diff_local_files(gold_session, image_name) 330 else: 331 logging.error('Given unhandled SkiaGoldSession StatusCode %s with error %s', status, error) 332 333 return FAIL 334 335 336def _get_batches(traces, batch_size): 337 for i in range(0, len(traces), batch_size): 338 yield traces[i:i + batch_size] 339 340 341def _get_gtest_filter_for_batch(args, batch): 342 expanded = ['%s%s' % (DEFAULT_TEST_PREFIX, trace) for trace in batch] 343 return '--gtest_filter=%s' % ':'.join(expanded) 344 345 346def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_results): 347 keys = get_skia_gold_keys(args, env) 348 349 with temporary_dir('angle_skia_gold_') as skia_gold_temp_dir: 350 gold_properties = angle_skia_gold_properties.ANGLESkiaGoldProperties(args) 351 gold_session_manager = angle_skia_gold_session_manager.ANGLESkiaGoldSessionManager( 352 skia_gold_temp_dir, gold_properties) 353 gold_session = gold_session_manager.GetSkiaGoldSession(keys, instance=args.instance) 354 355 traces = [trace.split(' ')[0] for trace in tests] 356 357 if args.isolated_script_test_filter: 358 traces = angle_test_util.FilterTests(traces, args.isolated_script_test_filter) 359 360 batches = _get_batches(traces, args.batch_size) 361 362 for batch in batches: 363 if angle_test_util.IsAndroid(): 364 android_helper.PrepareRestrictedTraces(batch) 365 366 for iteration in range(0, args.flaky_retries + 1): 367 # This is how we signal early exit 368 if not batch: 369 logging.debug('All tests in batch completed.') 370 break 371 if iteration > 0: 372 logging.info('Test run failed, running retry #%d...' % iteration) 373 374 gtest_filter = _get_gtest_filter_for_batch(args, batch) 375 cmd_args = [ 376 gtest_filter, 377 '--run-to-key-frame', 378 '--verbose-logging', 379 '--render-test-output-dir=%s' % screenshot_dir, 380 '--save-screenshots', 381 ] + extra_flags 382 if args.swiftshader: 383 cmd_args += ['--use-angle=swiftshader'] 384 385 logging.info('Running batch with args: %s' % cmd_args) 386 result, _, json_results = angle_test_util.RunTestSuite( 387 args.test_suite, cmd_args, env, use_xvfb=args.xvfb) 388 if result == 0: 389 batch_result = PASS 390 else: 391 batch_result = FAIL 392 logging.error('Batch FAIL! json_results: %s' % 393 json.dumps(json_results, indent=2)) 394 395 next_batch = [] 396 for trace in batch: 397 artifacts = {} 398 399 if batch_result == PASS: 400 test_name = DEFAULT_TEST_PREFIX + trace 401 if json_results['tests'][test_name]['actual'] == 'SKIP': 402 logging.info('Test skipped by suite: %s' % test_name) 403 result = SKIP 404 else: 405 logging.debug('upload test result: %s' % trace) 406 result = upload_test_result_to_skia_gold(args, gold_session_manager, 407 gold_session, gold_properties, 408 screenshot_dir, trace, 409 artifacts) 410 else: 411 result = batch_result 412 413 expected_result = SKIP if result == SKIP else PASS 414 test_results[trace] = {'expected': expected_result, 'actual': result} 415 if len(artifacts) > 0: 416 test_results[trace]['artifacts'] = artifacts 417 if result == FAIL: 418 next_batch.append(trace) 419 batch = next_batch 420 421 # These properties are recorded after iteration to ensure they only happen once. 422 for _, trace_results in test_results.items(): 423 result = trace_results['actual'] 424 results['num_failures_by_type'][result] += 1 425 if result == FAIL: 426 trace_results['is_unexpected'] = True 427 428 return results['num_failures_by_type'][FAIL] == 0 429 430 431def _shard_tests(tests, shard_count, shard_index): 432 return [tests[index] for index in range(shard_index, len(tests), shard_count)] 433 434 435def main(): 436 parser = argparse.ArgumentParser() 437 parser.add_argument('--isolated-script-test-output', type=str) 438 parser.add_argument('--isolated-script-test-perf-output', type=str) 439 parser.add_argument('-f', '--isolated-script-test-filter', '--filter', type=str) 440 parser.add_argument('--test-suite', help='Test suite to run.', default=DEFAULT_TEST_SUITE) 441 parser.add_argument('--render-test-output-dir', help='Directory to store screenshots') 442 parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') 443 parser.add_argument( 444 '--flaky-retries', help='Number of times to retry failed tests.', type=int, default=0) 445 parser.add_argument( 446 '--shard-count', 447 help='Number of shards for test splitting. Default is 1.', 448 type=int, 449 default=1) 450 parser.add_argument( 451 '--shard-index', 452 help='Index of the current shard for test splitting. Default is 0.', 453 type=int, 454 default=0) 455 parser.add_argument( 456 '--batch-size', 457 help='Number of tests to run in a group. Default: %d' % DEFAULT_BATCH_SIZE, 458 type=int, 459 default=DEFAULT_BATCH_SIZE) 460 parser.add_argument( 461 '-l', '--log', help='Log output level. Default is %s.' % DEFAULT_LOG, default=DEFAULT_LOG) 462 parser.add_argument('--swiftshader', help='Test with SwiftShader.', action='store_true') 463 parser.add_argument( 464 '-i', 465 '--instance', 466 '--gold-instance', 467 '--skia-gold-instance', 468 help='Skia Gold instance. Default is "%s".' % DEFAULT_GOLD_INSTANCE, 469 default=DEFAULT_GOLD_INSTANCE) 470 parser.add_argument( 471 '--use-permissive-pixel-comparison', 472 type=int, 473 help='Use a more permissive pixel comparison algorithm than the ' 474 'default "allow rounding errors" one. This is intended for use on CLs ' 475 'that are likely to cause differences in many tests, e.g. SwiftShader ' 476 'or driver changes. Can be enabled on bots by adding a ' 477 '"Use-Permissive-Angle-Pixel-Comparison: True" footer.') 478 479 add_skia_gold_args(parser) 480 481 args, extra_flags = parser.parse_known_args() 482 angle_test_util.SetupLogging(args.log.upper()) 483 484 env = os.environ.copy() 485 486 if angle_test_util.HasGtestShardsAndIndex(env): 487 args.shard_count, args.shard_index = angle_test_util.PopGtestShardsAndIndex(env) 488 489 angle_test_util.Initialize(args.test_suite) 490 491 results = { 492 'tests': {}, 493 'interrupted': False, 494 'seconds_since_epoch': time.time(), 495 'path_delimiter': '.', 496 'version': 3, 497 'num_failures_by_type': { 498 FAIL: 0, 499 PASS: 0, 500 SKIP: 0, 501 }, 502 } 503 504 test_results = {} 505 506 rc = 0 507 508 try: 509 # read test set 510 json_name = os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 511 'restricted_traces', 'restricted_traces.json') 512 with open(json_name) as fp: 513 tests = json.load(fp) 514 515 # Split tests according to sharding 516 sharded_tests = _shard_tests(tests['traces'], args.shard_count, args.shard_index) 517 518 if args.render_test_output_dir: 519 if not _run_tests(args, sharded_tests, extra_flags, env, args.render_test_output_dir, 520 results, test_results): 521 rc = 1 522 elif 'ISOLATED_OUTDIR' in env: 523 if not _run_tests(args, sharded_tests, extra_flags, env, env['ISOLATED_OUTDIR'], 524 results, test_results): 525 rc = 1 526 else: 527 with temporary_dir('angle_trace_') as temp_dir: 528 if not _run_tests(args, sharded_tests, extra_flags, env, temp_dir, results, 529 test_results): 530 rc = 1 531 532 except Exception: 533 traceback.print_exc() 534 results['interrupted'] = True 535 rc = 1 536 537 if test_results: 538 results['tests']['angle_restricted_trace_gold_tests'] = test_results 539 540 if args.isolated_script_test_output: 541 with open(args.isolated_script_test_output, 'w') as out_file: 542 out_file.write(json.dumps(results, indent=2)) 543 544 if args.isolated_script_test_perf_output: 545 with open(args.isolated_script_test_perf_output, 'w') as out_file: 546 out_file.write(json.dumps({})) 547 548 return rc 549 550 551if __name__ == '__main__': 552 sys.exit(main()) 553