1#! /usr/bin/env python3 2# 3# Copyright 2020 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7''' 8Script that re-captures the traces in the restricted trace folder. We can 9use this to update traces without needing to re-run the app on a device. 10''' 11 12import argparse 13import fnmatch 14import json 15import logging 16import os 17import pathlib 18import shutil 19import stat 20import subprocess 21import sys 22import tempfile 23import time 24 25from gen_restricted_traces import read_json as read_json, write_json as write_json 26from pathlib import Path 27 28from gen_restricted_traces import read_json as read_json 29 30SCRIPT_DIR = str(pathlib.Path(__file__).resolve().parent) 31PY_UTILS = str(pathlib.Path(SCRIPT_DIR) / '..' / 'py_utils') 32if PY_UTILS not in sys.path: 33 os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS) 34import android_helper 35import angle_test_util 36 37DEFAULT_TEST_SUITE = angle_test_util.ANGLE_TRACE_TEST_SUITE 38DEFAULT_TEST_JSON = 'restricted_traces.json' 39DEFAULT_LOG_LEVEL = 'info' 40DEFAULT_BACKUP_FOLDER = 'retrace-backups' 41 42EXIT_SUCCESS = 0 43EXIT_FAILURE = 1 44 45# Test expectations 46FAIL = 'FAIL' 47PASS = 'PASS' 48SKIP = 'SKIP' 49 50 51def get_trace_json_path(trace): 52 return os.path.join(get_script_dir(), trace, f'{trace}.json') 53 54 55def load_trace_json(trace): 56 json_file_name = get_trace_json_path(trace) 57 return read_json(json_file_name) 58 59 60def get_context(trace): 61 """Returns the trace context number.""" 62 json_data = load_trace_json(trace) 63 return str(json_data['WindowSurfaceContextID']) 64 65 66def get_script_dir(): 67 return os.path.dirname(sys.argv[0]) 68 69 70def context_header(trace, trace_path): 71 context_id = get_context(trace) 72 header = '%s_context%s.h' % (trace, context_id) 73 return os.path.join(trace_path, header) 74 75 76def src_trace_path(trace): 77 return os.path.join(get_script_dir(), trace) 78 79 80def get_num_frames(json_data): 81 metadata = json_data['TraceMetadata'] 82 return metadata['FrameEnd'] - metadata['FrameStart'] + 1 83 84 85def get_gles_version(json_data): 86 metadata = json_data['TraceMetadata'] 87 return (metadata['ContextClientMajorVersion'], metadata['ContextClientMinorVersion']) 88 89 90def set_gles_version(json_data, version): 91 metadata = json_data['TraceMetadata'] 92 metadata['ContextClientMajorVersion'] = version[0] 93 metadata['ContextClientMinorVersion'] = version[1] 94 95 96def save_trace_json(trace, data): 97 json_file_name = get_trace_json_path(trace) 98 return write_json(json_file_name, data) 99 100 101def path_contains_header(path): 102 if not os.path.isdir(path): 103 return False 104 for file in os.listdir(path): 105 if fnmatch.fnmatch(file, '*.h'): 106 return True 107 return False 108 109 110def chmod_directory(directory, perm): 111 assert os.path.isdir(directory) 112 for file in os.listdir(directory): 113 fn = os.path.join(directory, file) 114 os.chmod(fn, perm) 115 116 117def ensure_rmdir(directory): 118 if os.path.isdir(directory): 119 chmod_directory(directory, stat.S_IWRITE) 120 shutil.rmtree(directory) 121 122 123def copy_trace_folder(old_path, new_path): 124 logging.info('%s -> %s' % (old_path, new_path)) 125 ensure_rmdir(new_path) 126 shutil.copytree(old_path, new_path) 127 128 129def touch_trace_folder(trace_path): 130 for file in os.listdir(trace_path): 131 (Path(trace_path) / file).touch() 132 133 134def backup_single_trace(trace, backup_path): 135 trace_path = src_trace_path(trace) 136 trace_backup_path = os.path.join(backup_path, trace) 137 copy_trace_folder(trace_path, trace_backup_path) 138 139 140def backup_traces(args, traces): 141 for trace in angle_test_util.FilterTests(traces, args.traces): 142 backup_single_trace(trace, args.out_path) 143 144 145def restore_single_trace(trace, backup_path): 146 trace_path = src_trace_path(trace) 147 trace_backup_path = os.path.join(backup_path, trace) 148 if not os.path.isdir(trace_backup_path): 149 logging.error('Trace folder not found at %s' % trace_backup_path) 150 return False 151 else: 152 copy_trace_folder(trace_backup_path, trace_path) 153 touch_trace_folder(trace_path) 154 return True 155 156 157def restore_traces(args, traces): 158 for trace in angle_test_util.FilterTests(traces, args.traces): 159 restore_single_trace(trace, args.out_path) 160 161 162def run_autoninja(args): 163 autoninja_binary = 'autoninja' 164 if os.name == 'nt': 165 autoninja_binary += '.bat' 166 167 autoninja_args = [autoninja_binary, '-C', args.gn_path, args.test_suite] 168 logging.debug('Calling %s' % ' '.join(autoninja_args)) 169 if args.show_test_stdout: 170 subprocess.run(autoninja_args, check=True) 171 else: 172 subprocess.check_output(autoninja_args) 173 174 175def run_test_suite(args, trace_binary, trace, max_steps, additional_args, additional_env): 176 run_args = [ 177 angle_test_util.ExecutablePathInCurrentDir(trace_binary), 178 '--gtest_filter=TraceTest.%s' % trace, 179 '--max-steps-performed', 180 str(max_steps), 181 ] + additional_args 182 if not args.no_swiftshader: 183 run_args += ['--use-angle=swiftshader'] 184 185 env = {**os.environ.copy(), **additional_env} 186 env_string = ' '.join(['%s=%s' % item for item in additional_env.items()]) 187 if env_string: 188 env_string += ' ' 189 190 logging.info('%s%s' % (env_string, ' '.join(run_args))) 191 p = subprocess.run(run_args, env=env, capture_output=True, check=True) 192 if args.show_test_stdout: 193 logging.info('Test stdout:\n%s' % p.stdout.decode()) 194 195 196def upgrade_single_trace(args, trace_binary, trace, out_path, no_overwrite, c_sources): 197 logging.debug('Tracing %s' % trace) 198 199 trace_path = os.path.abspath(os.path.join(out_path, trace)) 200 if no_overwrite and path_contains_header(trace_path): 201 logging.info('Skipping "%s" because the out folder already exists' % trace) 202 return 203 204 json_data = load_trace_json(trace) 205 num_frames = get_num_frames(json_data) 206 207 metadata = json_data['TraceMetadata'] 208 logging.debug('Read metadata: %s' % str(metadata)) 209 210 max_steps = min(args.limit, num_frames) if args.limit else num_frames 211 212 # We start tracing from frame 2. --retrace-mode issues a Swap() after Setup() so we can 213 # accurately re-trace the MEC. 214 additional_env = { 215 'ANGLE_CAPTURE_LABEL': trace, 216 'ANGLE_CAPTURE_OUT_DIR': trace_path, 217 'ANGLE_CAPTURE_FRAME_START': '2', 218 'ANGLE_CAPTURE_FRAME_END': str(max_steps + 1), 219 } 220 if args.validation: 221 additional_env['ANGLE_CAPTURE_VALIDATION'] = '1' 222 # Also turn on shader output init to ensure we have no undefined values. 223 # This feature is also enabled in replay when using --validation. 224 additional_env[ 225 'ANGLE_FEATURE_OVERRIDES_ENABLED'] = 'allocateNonZeroMemory:forceInitShaderVariables' 226 if args.validation_expr: 227 additional_env['ANGLE_CAPTURE_VALIDATION_EXPR'] = args.validation_expr 228 if args.trim: 229 additional_env['ANGLE_CAPTURE_TRIM_ENABLED'] = '1' 230 if args.no_trim: 231 additional_env['ANGLE_CAPTURE_TRIM_ENABLED'] = '0' 232 # TODO: Remove when default. http://anglebug.com/7753 233 if c_sources: 234 additional_env['ANGLE_CAPTURE_SOURCE_EXT'] = 'c' 235 236 additional_args = ['--retrace-mode'] 237 238 try: 239 if not os.path.isdir(trace_path): 240 os.makedirs(trace_path) 241 242 run_test_suite(args, trace_binary, trace, max_steps, additional_args, additional_env) 243 244 json_file = "{}/{}.json".format(trace_path, trace) 245 if not os.path.exists(json_file): 246 logging.error( 247 f'There was a problem tracing "{trace}", could not find json file: {json_file}') 248 return False 249 except subprocess.CalledProcessError as e: 250 logging.exception('There was an exception running "%s":\n%s' % (trace, e.output.decode())) 251 return False 252 253 return True 254 255 256def upgrade_traces(args, traces): 257 run_autoninja(args) 258 trace_binary = os.path.join(args.gn_path, args.test_suite) 259 260 failures = [] 261 262 for trace in angle_test_util.FilterTests(traces, args.traces): 263 if not upgrade_single_trace(args, trace_binary, trace, args.out_path, args.no_overwrite, 264 args.c_sources): 265 failures += [trace] 266 267 if failures: 268 print('The following traces failed to upgrade:\n') 269 print('\n'.join([' ' + trace for trace in failures])) 270 return EXIT_FAILURE 271 272 return EXIT_SUCCESS 273 274 275def validate_single_trace(args, trace_binary, trace, additional_args, additional_env): 276 json_data = load_trace_json(trace) 277 num_frames = get_num_frames(json_data) 278 max_steps = min(args.limit, num_frames) if args.limit else num_frames 279 try: 280 run_test_suite(args, trace_binary, trace, max_steps, additional_args, additional_env) 281 except subprocess.CalledProcessError as e: 282 logging.error('There was a failure running "%s":\n%s' % (trace, e.output.decode())) 283 return False 284 return True 285 286 287def validate_traces(args, traces): 288 restore_traces(args, traces) 289 run_autoninja(args) 290 291 additional_args = ['--validation'] 292 additional_env = { 293 'ANGLE_FEATURE_OVERRIDES_ENABLED': 'allocateNonZeroMemory:forceInitShaderVariables' 294 } 295 296 failures = [] 297 trace_binary = os.path.join(args.gn_path, args.test_suite) 298 299 for trace in angle_test_util.FilterTests(traces, args.traces): 300 if not validate_single_trace(args, trace_binary, trace, additional_args, additional_env): 301 failures += [trace] 302 303 if failures: 304 print('The following traces failed to validate:\n') 305 print('\n'.join([' ' + trace for trace in failures])) 306 return EXIT_FAILURE 307 308 return EXIT_SUCCESS 309 310 311def interpret_traces(args, traces): 312 test_name = 'angle_trace_interpreter_tests' 313 results = { 314 'tests': { 315 test_name: {} 316 }, 317 'interrupted': False, 318 'seconds_since_epoch': time.time(), 319 'path_delimiter': '.', 320 'version': 3, 321 'num_failures_by_type': { 322 FAIL: 0, 323 PASS: 0, 324 SKIP: 0, 325 }, 326 } 327 328 if args.path: 329 trace_binary = os.path.join(args.path, args.test_suite) 330 else: 331 trace_binary = args.test_suite 332 333 for trace in angle_test_util.FilterTests(traces, args.traces): 334 with tempfile.TemporaryDirectory() as backup_path: 335 backup_single_trace(trace, backup_path) 336 result = FAIL 337 try: 338 with tempfile.TemporaryDirectory() as out_path: 339 logging.debug('Using temporary path %s.' % out_path) 340 if upgrade_single_trace(args, trace_binary, trace, out_path, False, True): 341 if restore_single_trace(trace, out_path): 342 validate_args = ['--trace-interpreter=c'] 343 if args.verbose: 344 validate_args += ['--verbose-logging'] 345 if validate_single_trace(args, trace_binary, trace, validate_args, {}): 346 logging.info('%s passed!' % trace) 347 result = PASS 348 finally: 349 restore_single_trace(trace, backup_path) 350 results['num_failures_by_type'][result] += 1 351 results['tests'][test_name][trace] = {'expected': PASS, 'actual': result} 352 353 if results['num_failures_by_type'][FAIL]: 354 logging.error('Some tests failed.') 355 return EXIT_FAILURE 356 357 if results['num_failures_by_type'][PASS] == 0: 358 logging.error('No tests ran. Please check your command line arguments.') 359 return EXIT_FAILURE 360 361 if args.test_output: 362 with open(args.test_output, 'w') as out_file: 363 out_file.write(json.dumps(results, indent=2)) 364 365 return EXIT_SUCCESS 366 367 368def add_upgrade_args(parser): 369 parser.add_argument( 370 '--validation', help='Enable state serialization validation calls.', action='store_true') 371 parser.add_argument( 372 '--validation-expr', 373 help='Validation expression, used to add more validation checkpoints.') 374 parser.add_argument( 375 '-L', 376 '--limit', 377 '--frame-limit', 378 type=int, 379 help='Limits the number of captured frames to produce a shorter trace than the original.') 380 parser.add_argument( 381 '--trim', action='store_true', help='Enables trace trimming. Breaks replay validation.') 382 parser.add_argument( 383 '--no-trim', action='store_true', help='Disables trace trimming. Useful for validation.') 384 parser.set_defaults(trim=True) 385 386 387def get_min_reqs(args, traces): 388 run_autoninja(args) 389 390 env = {} 391 default_args = ["--no-warmup"] 392 393 skipped_traces = [] 394 trace_binary = os.path.join(args.gn_path, args.test_suite) 395 396 for trace in angle_test_util.FilterTests(traces, args.traces): 397 print(f"Finding requirements for {trace}") 398 extensions = [] 399 json_data = load_trace_json(trace) 400 max_steps = get_num_frames(json_data) 401 402 # exts: a list of extensions to use with run_test_suite. If empty, 403 # then run_test_suite runs with all extensions enabled by default. 404 def run_test_suite_with_exts(exts): 405 additional_args = default_args.copy() 406 additional_args += ['--request-extensions', ' '.join(exts)] 407 408 try: 409 run_test_suite(args, trace_binary, trace, max_steps, additional_args, env) 410 except subprocess.CalledProcessError as error: 411 return False 412 return True 413 414 original_gles_version = get_gles_version(json_data) 415 original_extensions = None if 'RequiredExtensions' not in json_data else json_data[ 416 'RequiredExtensions'] 417 418 def restore_trace(): 419 if original_extensions is not None: 420 json_data['RequiredExtensions'] = original_extensions 421 set_gles_version(json_data, original_gles_version) 422 save_trace_json(trace, json_data) 423 424 try: 425 # Use the highest GLES version we have and empty the required 426 # extensions so that previous data doesn't affect the current 427 # run. 428 json_data['RequiredExtensions'] = [] 429 save_trace_json(trace, json_data) 430 if not run_test_suite_with_exts([]): 431 skipped_traces.append( 432 (trace, "Fails to run in default configuration on this machine")) 433 restore_trace() 434 continue 435 436 # Find minimum GLES version. 437 gles_versions = [(1, 0), (1, 1), (2, 0), (3, 0), (3, 1), (3, 2)] 438 min_version = None 439 for idx in range(len(gles_versions)): 440 min_version = gles_versions[idx] 441 set_gles_version(json_data, min_version) 442 save_trace_json(trace, json_data) 443 try: 444 run_test_suite(args, trace_binary, trace, max_steps, default_args, env) 445 except subprocess.CalledProcessError as error: 446 continue 447 break 448 449 # Get the list of requestable extensions for the GLES version. 450 try: 451 # Get the list of requestable extensions 452 with tempfile.NamedTemporaryFile() as tmp: 453 # Some operating systems will not allow a file to be open for writing 454 # by multiple processes. So close the temp file we just made before 455 # running the test suite. 456 tmp.close() 457 additional_args = ["--print-extensions-to-file", tmp.name] 458 run_test_suite(args, trace_binary, trace, max_steps, additional_args, env) 459 with open(tmp.name) as f: 460 for line in f: 461 extensions.append(line.strip()) 462 except Exception: 463 skipped_traces.append( 464 (trace, "Failed to read extension list, likely that test is skipped")) 465 restore_trace() 466 continue 467 468 if len(extensions) > 0 and not run_test_suite_with_exts(extensions): 469 skipped_traces.append((trace, "Requesting all extensions results in test failure")) 470 restore_trace() 471 continue 472 473 # Reset RequiredExtensions so it doesn't interfere with our search 474 json_data['RequiredExtensions'] = [] 475 save_trace_json(trace, json_data) 476 477 # Use a divide and conquer strategy to find the required extensions. 478 # Max depth is log(N) where N is the number of extensions. Expected 479 # runtime is p*log(N), where p is the number of required extensions. 480 # p*log(N) 481 # others: A list that contains one or more required extensions, 482 # but is not actively being searched 483 # exts: The list of extensions actively being searched 484 def recurse_run(others, exts, depth=0): 485 if len(exts) <= 1: 486 return exts 487 middle = int(len(exts) / 2) 488 left_partition = exts[:middle] 489 right_partition = exts[middle:] 490 left_passed = run_test_suite_with_exts(others + left_partition) 491 492 if depth > 0 and left_passed: 493 # We know right_passed must be False because one stack up 494 # run_test_suite(exts) returned False. 495 return recurse_run(others, left_partition) 496 497 right_passed = run_test_suite_with_exts(others + right_partition) 498 if left_passed and right_passed: 499 # Neither left nor right contain necessary extensions 500 return [] 501 elif left_passed: 502 # Only left contains necessary extensions 503 return recurse_run(others, left_partition, depth + 1) 504 elif right_passed: 505 # Only right contains necessary extensions 506 return recurse_run(others, right_partition, depth + 1) 507 else: 508 # Both left and right contain necessary extensions 509 left_reqs = recurse_run(others + right_partition, left_partition, depth + 1) 510 right_reqs = recurse_run(others + left_reqs, right_partition, depth + 1) 511 return left_reqs + right_reqs 512 513 recurse_reqs = recurse_run([], extensions, 0) 514 515 json_data['RequiredExtensions'] = recurse_reqs 516 save_trace_json(trace, json_data) 517 except BaseException as e: 518 restore_trace() 519 raise e 520 521 if skipped_traces: 522 print("Finished get_min_reqs, skipped traces:") 523 for trace, reason in skipped_traces: 524 print(f"\t{trace}: {reason}") 525 else: 526 print("Finished get_min_reqs for all traces specified") 527 528 529def main(): 530 parser = argparse.ArgumentParser() 531 parser.add_argument('-l', '--log', help='Logging level.', default=DEFAULT_LOG_LEVEL) 532 parser.add_argument( 533 '--test-suite', 534 help='Test Suite. Default is %s' % DEFAULT_TEST_SUITE, 535 default=DEFAULT_TEST_SUITE) 536 parser.add_argument( 537 '--no-swiftshader', 538 help='Trace against native Vulkan.', 539 action='store_true', 540 default=False) 541 parser.add_argument( 542 '--test-output', '--isolated-script-test-output', help='Where to write results JSON.') 543 544 subparsers = parser.add_subparsers(dest='command', required=True, help='Command to run.') 545 546 backup_parser = subparsers.add_parser( 547 'backup', help='Copies trace contents into a saved folder.') 548 backup_parser.add_argument( 549 'traces', help='Traces to back up. Supports fnmatch expressions.', default='*') 550 backup_parser.add_argument( 551 '-o', 552 '--out-path', 553 '--backup-path', 554 help='Destination folder. Default is "%s".' % DEFAULT_BACKUP_FOLDER, 555 default=DEFAULT_BACKUP_FOLDER) 556 557 restore_parser = subparsers.add_parser( 558 'restore', help='Copies traces from a saved folder to the trace folder.') 559 restore_parser.add_argument( 560 '-o', 561 '--out-path', 562 '--backup-path', 563 help='Path the traces were saved. Default is "%s".' % DEFAULT_BACKUP_FOLDER, 564 default=DEFAULT_BACKUP_FOLDER) 565 restore_parser.add_argument( 566 'traces', help='Traces to restore. Supports fnmatch expressions.', default='*') 567 568 upgrade_parser = subparsers.add_parser( 569 'upgrade', help='Re-trace existing traces, upgrading the format.') 570 upgrade_parser.add_argument('gn_path', help='GN build path') 571 upgrade_parser.add_argument('out_path', help='Output directory') 572 upgrade_parser.add_argument( 573 '-f', '--traces', '--filter', help='Trace filter. Defaults to all.', default='*') 574 upgrade_parser.add_argument( 575 '-n', 576 '--no-overwrite', 577 help='Skip traces which already exist in the out directory.', 578 action='store_true') 579 upgrade_parser.add_argument( 580 '-c', '--c-sources', help='Output to c sources instead of cpp.', action='store_true') 581 add_upgrade_args(upgrade_parser) 582 upgrade_parser.add_argument( 583 '--show-test-stdout', help='Log test output.', action='store_true', default=False) 584 585 validate_parser = subparsers.add_parser( 586 'validate', help='Runs the an updated test suite with validation enabled.') 587 validate_parser.add_argument('gn_path', help='GN build path') 588 validate_parser.add_argument('out_path', help='Path to the upgraded trace folder.') 589 validate_parser.add_argument( 590 'traces', help='Traces to validate. Supports fnmatch expressions.', default='*') 591 validate_parser.add_argument( 592 '-L', '--limit', '--frame-limit', type=int, help='Limits the number of tested frames.') 593 validate_parser.add_argument( 594 '--show-test-stdout', help='Log test output.', action='store_true', default=False) 595 596 interpret_parser = subparsers.add_parser( 597 'interpret', help='Complete trace interpreter self-test.') 598 interpret_parser.add_argument( 599 '-p', '--path', help='Path to trace executable. Default: look in CWD.') 600 interpret_parser.add_argument( 601 'traces', help='Traces to test. Supports fnmatch expressions.', default='*') 602 add_upgrade_args(interpret_parser) 603 interpret_parser.add_argument( 604 '--show-test-stdout', help='Log test output.', action='store_true', default=False) 605 interpret_parser.add_argument( 606 '-v', 607 '--verbose', 608 help='Verbose logging in the trace tests.', 609 action='store_true', 610 default=False) 611 612 get_min_reqs_parser = subparsers.add_parser( 613 'get_min_reqs', 614 help='Finds the minimum required extensions for a trace to successfully run.') 615 get_min_reqs_parser.add_argument('gn_path', help='GN build path') 616 get_min_reqs_parser.add_argument( 617 '--traces', 618 help='Traces to get minimum requirements for. Supports fnmatch expressions.', 619 default='*') 620 get_min_reqs_parser.add_argument( 621 '--show-test-stdout', help='Log test output.', action='store_true', default=False) 622 623 args, extra_flags = parser.parse_known_args() 624 625 logging.basicConfig(level=args.log.upper()) 626 627 # Load trace names 628 with open(os.path.join(get_script_dir(), DEFAULT_TEST_JSON)) as f: 629 traces = json.loads(f.read()) 630 631 traces = [trace.split(' ')[0] for trace in traces['traces']] 632 633 try: 634 if args.command == 'backup': 635 return backup_traces(args, traces) 636 elif args.command == 'restore': 637 return restore_traces(args, traces) 638 elif args.command == 'upgrade': 639 return upgrade_traces(args, traces) 640 elif args.command == 'validate': 641 return validate_traces(args, traces) 642 elif args.command == 'interpret': 643 return interpret_traces(args, traces) 644 elif args.command == 'get_min_reqs': 645 return get_min_reqs(args, traces) 646 else: 647 logging.fatal('Unknown command: %s' % args.command) 648 return EXIT_FAILURE 649 except subprocess.CalledProcessError as e: 650 if args.show_test_stdout: 651 logging.exception('There was an exception running "%s"' % traces) 652 else: 653 logging.exception('There was an exception running "%s": %s' % 654 (traces, e.output.decode())) 655 656 return EXIT_FAILURE 657 658 659if __name__ == '__main__': 660 sys.exit(main()) 661