1#! /usr/bin/env vpython3 2# 3# Copyright 2020 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7# restricted_trace_gold_tests.py: 8# Uses Skia Gold (https://skia.org/dev/testing/skiagold) to run pixel tests with ANGLE traces. 9# 10# Requires vpython to run standalone. Run with --help for usage instructions. 11 12import argparse 13import contextlib 14import json 15import logging 16import os 17import pathlib 18import platform 19import re 20import shutil 21import sys 22import tempfile 23import time 24import traceback 25 26 27PY_UTILS = str(pathlib.Path(__file__).resolve().parents[1] / 'py_utils') 28if PY_UTILS not in sys.path: 29 os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS) 30import android_helper 31import angle_path_util 32import angle_test_util 33from skia_gold import angle_skia_gold_properties 34from skia_gold import angle_skia_gold_session_manager 35 36angle_path_util.AddDepsDirToPath('testing/scripts') 37import common 38 39 40DEFAULT_TEST_SUITE = angle_test_util.ANGLE_TRACE_TEST_SUITE 41DEFAULT_TEST_PREFIX = 'TraceTest.' 42DEFAULT_SCREENSHOT_PREFIX = 'angle_vulkan_' 43SWIFTSHADER_SCREENSHOT_PREFIX = 'angle_vulkan_swiftshader_' 44DEFAULT_BATCH_SIZE = 5 45DEFAULT_LOG = 'info' 46DEFAULT_GOLD_INSTANCE = 'angle' 47 48# Test expectations 49FAIL = 'FAIL' 50PASS = 'PASS' 51SKIP = 'SKIP' 52 53 54@contextlib.contextmanager 55def temporary_dir(prefix=''): 56 path = tempfile.mkdtemp(prefix=prefix) 57 try: 58 yield path 59 finally: 60 logging.info("Removing temporary directory: %s" % path) 61 shutil.rmtree(path) 62 63 64def add_skia_gold_args(parser): 65 group = parser.add_argument_group('Skia Gold Arguments') 66 group.add_argument('--git-revision', help='Revision being tested.', default=None) 67 group.add_argument( 68 '--gerrit-issue', help='For Skia Gold integration. Gerrit issue ID.', default='') 69 group.add_argument( 70 '--gerrit-patchset', 71 help='For Skia Gold integration. Gerrit patch set number.', 72 default='') 73 group.add_argument( 74 '--buildbucket-id', help='For Skia Gold integration. Buildbucket build ID.', default='') 75 group.add_argument( 76 '--bypass-skia-gold-functionality', 77 action='store_true', 78 default=False, 79 help='Bypass all interaction with Skia Gold, effectively disabling the ' 80 'image comparison portion of any tests that use Gold. Only meant to ' 81 'be used in case a Gold outage occurs and cannot be fixed quickly.') 82 local_group = group.add_mutually_exclusive_group() 83 local_group.add_argument( 84 '--local-pixel-tests', 85 action='store_true', 86 default=None, 87 help='Specifies to run the test harness in local run mode or not. When ' 88 'run in local mode, uploading to Gold is disabled and links to ' 89 'help with local debugging are output. Running in local mode also ' 90 'implies --no-luci-auth. If both this and --no-local-pixel-tests are ' 91 'left unset, the test harness will attempt to detect whether it is ' 92 'running on a workstation or not and set this option accordingly.') 93 local_group.add_argument( 94 '--no-local-pixel-tests', 95 action='store_false', 96 dest='local_pixel_tests', 97 help='Specifies to run the test harness in non-local (bot) mode. When ' 98 'run in this mode, data is actually uploaded to Gold and triage links ' 99 'arge generated. If both this and --local-pixel-tests are left unset, ' 100 'the test harness will attempt to detect whether it is running on a ' 101 'workstation or not and set this option accordingly.') 102 group.add_argument( 103 '--no-luci-auth', 104 action='store_true', 105 default=False, 106 help='Don\'t use the service account provided by LUCI for ' 107 'authentication for Skia Gold, instead relying on gsutil to be ' 108 'pre-authenticated. Meant for testing locally instead of on the bots.') 109 110 111def run_angle_system_info_test(sysinfo_args, args, env): 112 with temporary_dir() as temp_dir: 113 sysinfo_args += ['--render-test-output-dir=' + temp_dir] 114 115 result, _, _ = angle_test_util.RunTestSuite( 116 'angle_system_info_test', sysinfo_args, env, use_xvfb=args.xvfb) 117 if result != 0: 118 raise Exception('Error getting system info.') 119 120 with open(os.path.join(temp_dir, 'angle_system_info.json')) as f: 121 return json.load(f) 122 123 124def to_hex(num): 125 return hex(int(num)) 126 127 128def to_hex_or_none(num): 129 return 'None' if num == None else to_hex(num) 130 131 132def to_non_empty_string_or_none(val): 133 return 'None' if val == '' else str(val) 134 135 136def to_non_empty_string_or_none_dict(d, key): 137 return 'None' if not key in d else to_non_empty_string_or_none(d[key]) 138 139 140def get_skia_gold_keys(args, env): 141 """Get all the JSON metadata that will be passed to golctl.""" 142 # All values need to be strings, otherwise goldctl fails. 143 144 # Only call this method one time 145 if hasattr(get_skia_gold_keys, 'called') and get_skia_gold_keys.called: 146 logging.exception('get_skia_gold_keys may only be called once') 147 get_skia_gold_keys.called = True 148 149 sysinfo_args = ['--vulkan', '-v'] 150 if args.swiftshader: 151 sysinfo_args.append('--swiftshader') 152 153 if angle_test_util.IsAndroid(): 154 json_data = android_helper.AngleSystemInfo(sysinfo_args) 155 logging.info(json_data) 156 os_name = 'Android' 157 os_version = android_helper.GetBuildFingerprint() 158 else: 159 json_data = run_angle_system_info_test(sysinfo_args, args, env) 160 os_name = to_non_empty_string_or_none(platform.system()) 161 os_version = to_non_empty_string_or_none(platform.version()) 162 163 if len(json_data.get('gpus', [])) == 0 or not 'activeGPUIndex' in json_data: 164 raise Exception('Error getting system info.') 165 166 active_gpu = json_data['gpus'][json_data['activeGPUIndex']] 167 168 angle_keys = { 169 'vendor_id': to_hex_or_none(active_gpu['vendorId']), 170 'device_id': to_hex_or_none(active_gpu['deviceId']), 171 'model_name': to_non_empty_string_or_none_dict(active_gpu, 'machineModelVersion'), 172 'manufacturer_name': to_non_empty_string_or_none_dict(active_gpu, 'machineManufacturer'), 173 'os': os_name, 174 'os_version': os_version, 175 'driver_version': to_non_empty_string_or_none_dict(active_gpu, 'driverVersion'), 176 'driver_vendor': to_non_empty_string_or_none_dict(active_gpu, 'driverVendor'), 177 } 178 179 return angle_keys 180 181 182def output_diff_local_files(gold_session, image_name): 183 """Logs the local diff image files from the given SkiaGoldSession. 184 185 Args: 186 gold_session: A skia_gold_session.SkiaGoldSession instance to pull files 187 from. 188 image_name: A string containing the name of the image/test that was 189 compared. 190 """ 191 given_file = gold_session.GetGivenImageLink(image_name) 192 closest_file = gold_session.GetClosestImageLink(image_name) 193 diff_file = gold_session.GetDiffImageLink(image_name) 194 failure_message = 'Unable to retrieve link' 195 logging.error('Generated image: %s', given_file or failure_message) 196 logging.error('Closest image: %s', closest_file or failure_message) 197 logging.error('Diff image: %s', diff_file or failure_message) 198 199 200def get_trace_key_frame(trace): 201 # read trace info 202 json_name = os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 'restricted_traces', 203 trace, trace + '.json') 204 with open(json_name) as fp: 205 trace_info = json.load(fp) 206 207 # Check its metadata for a keyframe 208 keyframe = '' 209 if 'KeyFrames' in trace_info['TraceMetadata']: 210 # KeyFrames is an array, but we only use the first value for now 211 keyframe = str(trace_info['TraceMetadata']['KeyFrames'][0]) 212 logging.info('trace %s is using a keyframe of %s' % (trace, keyframe)) 213 214 return keyframe 215 216 217def upload_test_result_to_skia_gold(args, gold_session_manager, gold_session, gold_properties, 218 screenshot_dir, trace, artifacts): 219 """Compares the given image using Skia Gold and uploads the result. 220 221 No uploading is done if the test is being run in local run mode. Compares 222 the given screenshot to baselines provided by Gold, raising an Exception if 223 a match is not found. 224 225 Args: 226 args: Command line options. 227 gold_session_manager: Skia Gold session manager. 228 gold_session: Skia Gold session. 229 gold_properties: Skia Gold properties. 230 screenshot_dir: directory where the test stores screenshots. 231 trace: base name of the trace being checked. 232 artifacts: dictionary of JSON artifacts to pass to the result merger. 233 """ 234 235 use_luci = not (gold_properties.local_pixel_tests or gold_properties.no_luci_auth) 236 237 # Determine if this trace is using a keyframe 238 image_name = trace 239 keyframe = get_trace_key_frame(trace) 240 if keyframe != '': 241 image_name = trace + '_frame' + keyframe 242 logging.debug('Using %s as image_name for upload' % image_name) 243 244 # Note: this would be better done by iterating the screenshot directory. 245 prefix = SWIFTSHADER_SCREENSHOT_PREFIX if args.swiftshader else DEFAULT_SCREENSHOT_PREFIX 246 png_file_name = os.path.join(screenshot_dir, prefix + image_name + '.png') 247 248 if not os.path.isfile(png_file_name): 249 raise Exception('Screenshot not found: ' + png_file_name) 250 251 if args.use_permissive_pixel_comparison: 252 # These arguments cause Gold to use the sample area inexact matching 253 # algorithm. It is set to allow any of up to 3 pixels in each 4x4 group 254 # of pixels to differ by any amount. Pixels that differ by a max of 1 255 # on all channels (i.e. have differences that can be attributed to 256 # rounding errors) do not count towards this limit. 257 # 258 # An image that passes due to this logic is auto-approved as a new good 259 # image. 260 inexact_matching_args = [ 261 '--add-test-optional-key', 262 'image_matching_algorithm:sample_area', 263 '--add-test-optional-key', 264 'sample_area_width:4', 265 '--add-test-optional-key', 266 'sample_area_max_different_pixels_per_area:3', 267 '--add-test-optional-key', 268 'sample_area_channel_delta_threshold:1', 269 ] 270 else: 271 # These arguments cause Gold to use the fuzzy inexact matching 272 # algorithm. It is set to allow up to 20k pixels to differ by 1 on all 273 # channels, which is meant to help reduce triage overhead caused by new 274 # images from rounding differences. 275 # 276 # The max number of pixels is fairly arbitrary, but the diff threshold 277 # is intentional since we don't want to let in any changes that can't be 278 # attributed to rounding errors. 279 # 280 # An image that passes due to this logic is auto-approved as a new good 281 # image. 282 inexact_matching_args = [ 283 '--add-test-optional-key', 284 'image_matching_algorithm:fuzzy', 285 '--add-test-optional-key', 286 'fuzzy_max_different_pixels:20000', 287 '--add-test-optional-key', 288 'fuzzy_pixel_per_channel_delta_threshold:1', 289 ] 290 291 status, error = gold_session.RunComparison( 292 name=image_name, 293 png_file=png_file_name, 294 use_luci=use_luci, 295 inexact_matching_args=inexact_matching_args) 296 297 artifact_name = os.path.basename(png_file_name) 298 artifacts[artifact_name] = [artifact_name] 299 300 if not status: 301 return PASS 302 303 status_codes = gold_session_manager.GetSessionClass().StatusCodes 304 if status == status_codes.AUTH_FAILURE: 305 logging.error('Gold authentication failed with output %s', error) 306 elif status == status_codes.INIT_FAILURE: 307 logging.error('Gold initialization failed with output %s', error) 308 elif status == status_codes.COMPARISON_FAILURE_REMOTE: 309 _, triage_link = gold_session.GetTriageLinks(image_name) 310 if not triage_link: 311 logging.error('Failed to get triage link for %s, raw output: %s', image_name, error) 312 logging.error('Reason for no triage link: %s', 313 gold_session.GetTriageLinkOmissionReason(image_name)) 314 if gold_properties.IsTryjobRun(): 315 # Pick "show all results" so we can see the tryjob images by default. 316 triage_link += '&master=true' 317 artifacts['triage_link_for_entire_cl'] = [triage_link] 318 else: 319 artifacts['gold_triage_link'] = [triage_link] 320 elif status == status_codes.COMPARISON_FAILURE_LOCAL: 321 logging.error('Local comparison failed. Local diff files:') 322 output_diff_local_files(gold_session, image_name) 323 elif status == status_codes.LOCAL_DIFF_FAILURE: 324 logging.error( 325 'Local comparison failed and an error occurred during diff ' 326 'generation: %s', error) 327 # There might be some files, so try outputting them. 328 logging.error('Local diff files:') 329 output_diff_local_files(gold_session, image_name) 330 else: 331 logging.error('Given unhandled SkiaGoldSession StatusCode %s with error %s', status, error) 332 333 return FAIL 334 335 336def _get_batches(traces, batch_size): 337 for i in range(0, len(traces), batch_size): 338 yield traces[i:i + batch_size] 339 340 341def _get_gtest_filter_for_batch(args, batch): 342 expanded = ['%s%s' % (DEFAULT_TEST_PREFIX, trace) for trace in batch] 343 return '--gtest_filter=%s' % ':'.join(expanded) 344 345 346def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_results): 347 keys = get_skia_gold_keys(args, env) 348 349 if angle_test_util.IsAndroid() and args.test_suite == DEFAULT_TEST_SUITE: 350 android_helper.RunSmokeTest() 351 352 with temporary_dir('angle_skia_gold_') as skia_gold_temp_dir: 353 gold_properties = angle_skia_gold_properties.ANGLESkiaGoldProperties(args) 354 gold_session_manager = angle_skia_gold_session_manager.ANGLESkiaGoldSessionManager( 355 skia_gold_temp_dir, gold_properties) 356 gold_session = gold_session_manager.GetSkiaGoldSession(keys, instance=args.instance) 357 358 traces = [trace.split(' ')[0] for trace in tests] 359 360 if args.isolated_script_test_filter: 361 traces = angle_test_util.FilterTests(traces, args.isolated_script_test_filter) 362 363 batches = _get_batches(traces, args.batch_size) 364 365 for batch in batches: 366 if angle_test_util.IsAndroid(): 367 android_helper.PrepareRestrictedTraces(batch) 368 369 for iteration in range(0, args.flaky_retries + 1): 370 # This is how we signal early exit 371 if not batch: 372 logging.debug('All tests in batch completed.') 373 break 374 if iteration > 0: 375 logging.info('Test run failed, running retry #%d...' % iteration) 376 377 gtest_filter = _get_gtest_filter_for_batch(args, batch) 378 cmd_args = [ 379 gtest_filter, 380 '--run-to-key-frame', 381 '--verbose-logging', 382 '--render-test-output-dir=%s' % screenshot_dir, 383 '--save-screenshots', 384 ] + extra_flags 385 if args.swiftshader: 386 cmd_args += ['--use-angle=swiftshader'] 387 388 logging.info('Running batch with args: %s' % cmd_args) 389 result, _, json_results = angle_test_util.RunTestSuite( 390 args.test_suite, cmd_args, env, use_xvfb=args.xvfb) 391 if result == 0: 392 batch_result = PASS 393 else: 394 batch_result = FAIL 395 logging.error('Batch FAIL! json_results: %s' % 396 json.dumps(json_results, indent=2)) 397 398 next_batch = [] 399 for trace in batch: 400 artifacts = {} 401 402 if batch_result == PASS: 403 test_name = DEFAULT_TEST_PREFIX + trace 404 if json_results['tests'][test_name]['actual'] == 'SKIP': 405 logging.info('Test skipped by suite: %s' % test_name) 406 result = SKIP 407 else: 408 logging.debug('upload test result: %s' % trace) 409 result = upload_test_result_to_skia_gold(args, gold_session_manager, 410 gold_session, gold_properties, 411 screenshot_dir, trace, 412 artifacts) 413 else: 414 result = batch_result 415 416 expected_result = SKIP if result == SKIP else PASS 417 test_results[trace] = {'expected': expected_result, 'actual': result} 418 if len(artifacts) > 0: 419 test_results[trace]['artifacts'] = artifacts 420 if result == FAIL: 421 next_batch.append(trace) 422 batch = next_batch 423 424 # These properties are recorded after iteration to ensure they only happen once. 425 for _, trace_results in test_results.items(): 426 result = trace_results['actual'] 427 results['num_failures_by_type'][result] += 1 428 if result == FAIL: 429 trace_results['is_unexpected'] = True 430 431 return results['num_failures_by_type'][FAIL] == 0 432 433 434def _shard_tests(tests, shard_count, shard_index): 435 return [tests[index] for index in range(shard_index, len(tests), shard_count)] 436 437 438def main(): 439 parser = argparse.ArgumentParser() 440 parser.add_argument('--isolated-script-test-output', type=str) 441 parser.add_argument('--isolated-script-test-perf-output', type=str) 442 parser.add_argument('-f', '--isolated-script-test-filter', '--filter', type=str) 443 parser.add_argument('--test-suite', help='Test suite to run.', default=DEFAULT_TEST_SUITE) 444 parser.add_argument('--render-test-output-dir', help='Directory to store screenshots') 445 parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') 446 parser.add_argument( 447 '--flaky-retries', help='Number of times to retry failed tests.', type=int, default=0) 448 parser.add_argument( 449 '--shard-count', 450 help='Number of shards for test splitting. Default is 1.', 451 type=int, 452 default=1) 453 parser.add_argument( 454 '--shard-index', 455 help='Index of the current shard for test splitting. Default is 0.', 456 type=int, 457 default=0) 458 parser.add_argument( 459 '--batch-size', 460 help='Number of tests to run in a group. Default: %d' % DEFAULT_BATCH_SIZE, 461 type=int, 462 default=DEFAULT_BATCH_SIZE) 463 parser.add_argument( 464 '-l', '--log', help='Log output level. Default is %s.' % DEFAULT_LOG, default=DEFAULT_LOG) 465 parser.add_argument('--swiftshader', help='Test with SwiftShader.', action='store_true') 466 parser.add_argument( 467 '-i', 468 '--instance', 469 '--gold-instance', 470 '--skia-gold-instance', 471 help='Skia Gold instance. Default is "%s".' % DEFAULT_GOLD_INSTANCE, 472 default=DEFAULT_GOLD_INSTANCE) 473 parser.add_argument( 474 '--use-permissive-pixel-comparison', 475 type=int, 476 help='Use a more permissive pixel comparison algorithm than the ' 477 'default "allow rounding errors" one. This is intended for use on CLs ' 478 'that are likely to cause differences in many tests, e.g. SwiftShader ' 479 'or driver changes. Can be enabled on bots by adding a ' 480 '"Use-Permissive-Angle-Pixel-Comparison: True" footer.') 481 482 add_skia_gold_args(parser) 483 484 args, extra_flags = parser.parse_known_args() 485 angle_test_util.SetupLogging(args.log.upper()) 486 487 env = os.environ.copy() 488 489 if angle_test_util.HasGtestShardsAndIndex(env): 490 args.shard_count, args.shard_index = angle_test_util.PopGtestShardsAndIndex(env) 491 492 angle_test_util.Initialize(args.test_suite) 493 494 results = { 495 'tests': {}, 496 'interrupted': False, 497 'seconds_since_epoch': time.time(), 498 'path_delimiter': '.', 499 'version': 3, 500 'num_failures_by_type': { 501 FAIL: 0, 502 PASS: 0, 503 SKIP: 0, 504 }, 505 } 506 507 test_results = {} 508 509 rc = 0 510 511 try: 512 # read test set 513 json_name = os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 514 'restricted_traces', 'restricted_traces.json') 515 with open(json_name) as fp: 516 tests = json.load(fp) 517 518 # Split tests according to sharding 519 sharded_tests = _shard_tests(tests['traces'], args.shard_count, args.shard_index) 520 521 if args.render_test_output_dir: 522 if not _run_tests(args, sharded_tests, extra_flags, env, args.render_test_output_dir, 523 results, test_results): 524 rc = 1 525 elif 'ISOLATED_OUTDIR' in env: 526 if not _run_tests(args, sharded_tests, extra_flags, env, env['ISOLATED_OUTDIR'], 527 results, test_results): 528 rc = 1 529 else: 530 with temporary_dir('angle_trace_') as temp_dir: 531 if not _run_tests(args, sharded_tests, extra_flags, env, temp_dir, results, 532 test_results): 533 rc = 1 534 535 except Exception: 536 traceback.print_exc() 537 results['interrupted'] = True 538 rc = 1 539 540 if test_results: 541 results['tests']['angle_restricted_trace_gold_tests'] = test_results 542 543 if args.isolated_script_test_output: 544 with open(args.isolated_script_test_output, 'w') as out_file: 545 out_file.write(json.dumps(results, indent=2)) 546 547 if args.isolated_script_test_perf_output: 548 with open(args.isolated_script_test_perf_output, 'w') as out_file: 549 out_file.write(json.dumps({})) 550 551 return rc 552 553 554if __name__ == '__main__': 555 sys.exit(main()) 556