1#! /usr/bin/env vpython3 2# 3# Copyright 2020 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7""" 8Script testing capture_replay with angle_end2end_tests 9""" 10 11# Automation script will: 12# 1. Build all tests in angle_end2end with frame capture enabled 13# 2. Run each test with frame capture 14# 3. Build CaptureReplayTest with cpp trace files 15# 4. Run CaptureReplayTest 16# 5. Output the number of test successes and failures. A test succeeds if no error occurs during 17# its capture and replay, and the GL states at the end of two runs match. Any unexpected failure 18# will return non-zero exit code 19 20# Run this script with Python to test capture replay on angle_end2end tests 21# python path/to/capture_replay_tests.py 22# Command line arguments: run with --help for a full list. 23 24import argparse 25import difflib 26import distutils.util 27import fnmatch 28import getpass 29import json 30import logging 31import math 32import multiprocessing 33import os 34import queue 35import re 36import shutil 37import subprocess 38import sys 39import tempfile 40import time 41import traceback 42 43PIPE_STDOUT = True 44DEFAULT_OUT_DIR = "out/CaptureReplayTest" # relative to angle folder 45DEFAULT_FILTER = "*/ES2_Vulkan_SwiftShader" 46DEFAULT_TEST_SUITE = "angle_end2end_tests" 47REPLAY_SAMPLE_FOLDER = "src/tests/capture_replay_tests" # relative to angle folder 48DEFAULT_BATCH_COUNT = 8 # number of tests batched together 49TRACE_FILE_SUFFIX = "_context" # because we only deal with 1 context right now 50RESULT_TAG = "*RESULT" 51STATUS_MESSAGE_PERIOD = 20 # in seconds 52SUBPROCESS_TIMEOUT = 600 # in seconds 53DEFAULT_RESULT_FILE = "results.txt" 54DEFAULT_LOG_LEVEL = "info" 55DEFAULT_MAX_JOBS = 8 56DEFAULT_MAX_NINJA_JOBS = 1 57REPLAY_BINARY = "capture_replay_tests" 58if sys.platform == "win32": 59 REPLAY_BINARY += ".exe" 60TRACE_FOLDER = "traces" 61 62EXIT_SUCCESS = 0 63EXIT_FAILURE = 1 64REPLAY_INITIALIZATION_FAILURE = -1 65REPLAY_SERIALIZATION_FAILURE = -2 66 67switch_case_without_return_template = """\ 68 case {case}: 69 {namespace}::{call}({params}); 70 break; 71""" 72 73switch_case_with_return_template = """\ 74 case {case}: 75 return {namespace}::{call}({params}); 76""" 77 78default_case_without_return_template = """\ 79 default: 80 break;""" 81default_case_with_return_template = """\ 82 default: 83 return {default_val};""" 84 85 86def winext(name, ext): 87 return ("%s.%s" % (name, ext)) if sys.platform == "win32" else name 88 89class SubProcess(): 90 91 def __init__(self, command, logger, env=os.environ, pipe_stdout=PIPE_STDOUT): 92 # shell=False so that only 1 subprocess is spawned. 93 # if shell=True, a shell process is spawned, which in turn spawns the process running 94 # the command. Since we do not have a handle to the 2nd process, we cannot terminate it. 95 if pipe_stdout: 96 self.proc_handle = subprocess.Popen( 97 command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) 98 else: 99 self.proc_handle = subprocess.Popen(command, env=env, shell=False) 100 self._logger = logger 101 102 def Join(self, timeout): 103 self._logger.debug('Joining with subprocess %d, timeout %s' % (self.Pid(), str(timeout))) 104 output = self.proc_handle.communicate(timeout=timeout)[0] 105 if output: 106 output = output.decode('utf-8') 107 else: 108 output = '' 109 return self.proc_handle.returncode, output 110 111 def Pid(self): 112 return self.proc_handle.pid 113 114 def Kill(self): 115 self.proc_handle.terminate() 116 self.proc_handle.wait() 117 118 119# class that manages all child processes of a process. Any process thats spawns subprocesses 120# should have this. This object is created inside the main process, and each worker process. 121class ChildProcessesManager(): 122 123 @classmethod 124 def _GetGnAbsolutePaths(self): 125 return os.path.join('third_party', 'depot_tools', winext('gn', 'bat')) 126 127 @classmethod 128 def _GetAutoNinjaAbsolutePaths(self): 129 return os.path.join('third_party', 'depot_tools', 'autoninja.py') 130 131 def __init__(self, args, logger, ninja_lock): 132 # a dictionary of Subprocess, with pid as key 133 self.subprocesses = {} 134 # list of Python multiprocess.Process handles 135 self.workers = [] 136 137 self._gn_path = self._GetGnAbsolutePaths() 138 self._autoninja_path = self._GetAutoNinjaAbsolutePaths() 139 self._logger = logger 140 self._ninja_lock = ninja_lock 141 self.runtimes = {} 142 self._args = args 143 144 def RunSubprocess(self, command, env=None, pipe_stdout=True, timeout=None): 145 proc = SubProcess(command, self._logger, env, pipe_stdout) 146 self._logger.debug('Created subprocess: %s with pid %d' % (' '.join(command), proc.Pid())) 147 self.subprocesses[proc.Pid()] = proc 148 start_time = time.time() 149 try: 150 returncode, output = self.subprocesses[proc.Pid()].Join(timeout) 151 elapsed_time = time.time() - start_time 152 cmd_name = os.path.basename(command[0]) 153 self.runtimes.setdefault(cmd_name, 0.0) 154 self.runtimes[cmd_name] += elapsed_time 155 self.RemoveSubprocess(proc.Pid()) 156 if returncode != 0: 157 return -1, output 158 return returncode, output 159 except KeyboardInterrupt: 160 raise 161 except subprocess.TimeoutExpired as e: 162 self.RemoveSubprocess(proc.Pid()) 163 return -2, str(e) 164 except Exception as e: 165 self.RemoveSubprocess(proc.Pid()) 166 return -1, str(e) 167 168 def RemoveSubprocess(self, subprocess_id): 169 assert subprocess_id in self.subprocesses 170 self.subprocesses[subprocess_id].Kill() 171 del self.subprocesses[subprocess_id] 172 173 def AddWorker(self, worker): 174 self.workers.append(worker) 175 176 def KillAll(self): 177 for subprocess_id in self.subprocesses: 178 self.subprocesses[subprocess_id].Kill() 179 for worker in self.workers: 180 worker.terminate() 181 worker.join() 182 worker.close() # to release file descriptors immediately 183 self.subprocesses = {} 184 self.workers = [] 185 186 def JoinWorkers(self): 187 for worker in self.workers: 188 worker.join() 189 worker.close() 190 self.workers = [] 191 192 def IsAnyWorkerAlive(self): 193 return any([worker.is_alive() for worker in self.workers]) 194 195 def GetRemainingWorkers(self): 196 count = 0 197 for worker in self.workers: 198 if worker.is_alive(): 199 count += 1 200 return count 201 202 def RunGNGen(self, build_dir, pipe_stdout, extra_gn_args=[]): 203 gn_args = [('angle_with_capture_by_default', 'true')] + extra_gn_args 204 if self._args.use_reclient: 205 gn_args.append(('use_remoteexec', 'true')) 206 if not self._args.debug: 207 gn_args.append(('is_debug', 'false')) 208 gn_args.append(('symbol_level', '1')) 209 gn_args.append(('angle_assert_always_on', 'true')) 210 if self._args.asan: 211 gn_args.append(('is_asan', 'true')) 212 args_str = ' '.join(['%s=%s' % (k, v) for (k, v) in gn_args]) 213 cmd = [self._gn_path, 'gen', '--args=%s' % args_str, build_dir] 214 self._logger.info(' '.join(cmd)) 215 return self.RunSubprocess(cmd, pipe_stdout=pipe_stdout) 216 217 def RunAutoNinja(self, build_dir, target, pipe_stdout): 218 cmd = [sys.executable, self._autoninja_path, '-C', build_dir, target] 219 with self._ninja_lock: 220 self._logger.info(' '.join(cmd)) 221 return self.RunSubprocess(cmd, pipe_stdout=pipe_stdout) 222 223 224def GetTestsListForFilter(args, test_path, filter, logger): 225 cmd = GetRunCommand(args, test_path) + ["--list-tests", "--gtest_filter=%s" % filter] 226 logger.info('Getting test list from "%s"' % " ".join(cmd)) 227 return subprocess.check_output(cmd, text=True) 228 229 230def ParseTestNamesFromTestList(output, test_expectation, also_run_skipped_for_capture_tests, 231 logger): 232 output_lines = output.splitlines() 233 tests = [] 234 seen_start_of_tests = False 235 disabled = 0 236 for line in output_lines: 237 l = line.strip() 238 if l == 'Tests list:': 239 seen_start_of_tests = True 240 elif l == 'End tests list.': 241 break 242 elif not seen_start_of_tests: 243 pass 244 elif not test_expectation.TestIsSkippedForCapture(l) or also_run_skipped_for_capture_tests: 245 tests.append(l) 246 else: 247 disabled += 1 248 249 logger.info('Found %s tests and %d disabled tests.' % (len(tests), disabled)) 250 return tests 251 252 253def GetRunCommand(args, command): 254 if args.xvfb: 255 return ['vpython', 'testing/xvfb.py', command] 256 else: 257 return [command] 258 259 260class GroupedResult(): 261 Passed = "Pass" 262 Failed = "Fail" 263 TimedOut = "Timeout" 264 CompileFailed = "CompileFailed" 265 CaptureFailed = "CaptureFailed" 266 ReplayFailed = "ReplayFailed" 267 Skipped = "Skipped" 268 FailedToTrace = "FailedToTrace" 269 270 ResultTypes = [ 271 Passed, Failed, TimedOut, CompileFailed, CaptureFailed, ReplayFailed, Skipped, 272 FailedToTrace 273 ] 274 275 def __init__(self, resultcode, message, output, tests): 276 self.resultcode = resultcode 277 self.message = message 278 self.output = output 279 self.tests = [] 280 for test in tests: 281 self.tests.append(test) 282 283 284def BatchName(batch_or_result): 285 return 'batch_%03d' % batch_or_result.batch_index 286 287 288class TestBatchResult(): 289 290 display_output_lines = 20 291 292 def __init__(self, batch_index, grouped_results, verbose): 293 self.batch_index = batch_index 294 self.results = {} 295 for result_type in GroupedResult.ResultTypes: 296 self.results[result_type] = [] 297 298 for grouped_result in grouped_results: 299 for test in grouped_result.tests: 300 self.results[grouped_result.resultcode].append(test.full_test_name) 301 302 self.repr_str = "" 303 self.GenerateRepresentationString(grouped_results, verbose) 304 305 def __str__(self): 306 return self.repr_str 307 308 def GenerateRepresentationString(self, grouped_results, verbose): 309 self.repr_str += BatchName(self) + "\n" 310 for grouped_result in grouped_results: 311 self.repr_str += grouped_result.resultcode + ": " + grouped_result.message + "\n" 312 for test in grouped_result.tests: 313 self.repr_str += "\t" + test.full_test_name + "\n" 314 if verbose: 315 self.repr_str += grouped_result.output 316 else: 317 if grouped_result.resultcode == GroupedResult.CompileFailed: 318 self.repr_str += TestBatchResult.ExtractErrors(grouped_result.output) 319 elif grouped_result.resultcode != GroupedResult.Passed: 320 self.repr_str += grouped_result.output 321 322 def ExtractErrors(output): 323 lines = output.splitlines() 324 error_lines = [] 325 for i in range(len(lines)): 326 if ": error:" in lines[i]: 327 error_lines.append(lines[i] + "\n") 328 if i + 1 < len(lines): 329 error_lines.append(lines[i + 1] + "\n") 330 return "".join(error_lines) 331 332 333class Test(): 334 335 def __init__(self, test_name): 336 self.full_test_name = test_name 337 self.params = test_name.split('/')[1] 338 self.context_id = 0 339 self.test_index = -1 # index of test within a test batch 340 self._label = self.full_test_name.replace(".", "_").replace("/", "_") 341 self.skipped_by_suite = False 342 343 def __str__(self): 344 return self.full_test_name + " Params: " + self.params 345 346 def GetLabel(self): 347 return self._label 348 349 def CanRunReplay(self, trace_folder_path): 350 test_files = [] 351 label = self.GetLabel() 352 assert (self.context_id == 0) 353 for f in os.listdir(trace_folder_path): 354 if os.path.isfile(os.path.join(trace_folder_path, f)) and f.startswith(label): 355 test_files.append(f) 356 frame_files_count = 0 357 context_header_count = 0 358 context_source_count = 0 359 source_json_count = 0 360 context_id = 0 361 for f in test_files: 362 # TODO: Consolidate. http://anglebug.com/7753 363 if "_001.cpp" in f or "_001.c" in f: 364 frame_files_count += 1 365 elif f.endswith(".json"): 366 source_json_count += 1 367 elif f.endswith(".h"): 368 context_header_count += 1 369 if TRACE_FILE_SUFFIX in f: 370 context = f.split(TRACE_FILE_SUFFIX)[1][:-2] 371 context_id = int(context) 372 # TODO: Consolidate. http://anglebug.com/7753 373 elif f.endswith(".cpp") or f.endswith(".c"): 374 context_source_count += 1 375 can_run_replay = frame_files_count >= 1 and context_header_count >= 1 \ 376 and context_source_count >= 1 and source_json_count == 1 377 if not can_run_replay: 378 return False 379 self.context_id = context_id 380 return True 381 382 383def _FormatEnv(env): 384 return ' '.join(['%s=%s' % (k, v) for (k, v) in env.items()]) 385 386 387class TestBatch(): 388 389 CAPTURE_FRAME_END = 100 390 391 def __init__(self, args, logger, batch_index): 392 self.args = args 393 self.tests = [] 394 self.results = [] 395 self.logger = logger 396 self.batch_index = batch_index 397 398 def SetWorkerId(self, worker_id): 399 self.trace_dir = "%s%d" % (TRACE_FOLDER, worker_id) 400 self.trace_folder_path = os.path.join(REPLAY_SAMPLE_FOLDER, self.trace_dir) 401 402 def RunWithCapture(self, args, child_processes_manager): 403 test_exe_path = os.path.join(args.out_dir, 'Capture', args.test_suite) 404 405 extra_env = { 406 'ANGLE_CAPTURE_SERIALIZE_STATE': '1', 407 'ANGLE_FEATURE_OVERRIDES_ENABLED': 'forceRobustResourceInit:forceInitShaderVariables', 408 'ANGLE_FEATURE_OVERRIDES_DISABLED': 'supportsHostImageCopy', 409 'ANGLE_CAPTURE_ENABLED': '1', 410 'ANGLE_CAPTURE_OUT_DIR': self.trace_folder_path, 411 } 412 413 if args.mec > 0: 414 extra_env['ANGLE_CAPTURE_FRAME_START'] = '{}'.format(args.mec) 415 extra_env['ANGLE_CAPTURE_FRAME_END'] = '{}'.format(args.mec + 1) 416 else: 417 extra_env['ANGLE_CAPTURE_FRAME_END'] = '{}'.format(self.CAPTURE_FRAME_END) 418 419 if args.expose_nonconformant_features: 420 extra_env[ 421 'ANGLE_FEATURE_OVERRIDES_ENABLED'] += ':exposeNonConformantExtensionsAndVersions' 422 423 env = {**os.environ.copy(), **extra_env} 424 425 if not self.args.keep_temp_files: 426 ClearFolderContent(self.trace_folder_path) 427 filt = ':'.join([test.full_test_name for test in self.tests]) 428 429 cmd = GetRunCommand(args, test_exe_path) 430 results_file = tempfile.mktemp() 431 cmd += [ 432 '--gtest_filter=%s' % filt, 433 '--angle-per-test-capture-label', 434 '--results-file=' + results_file, 435 ] 436 self.logger.info('%s %s' % (_FormatEnv(extra_env), ' '.join(cmd))) 437 438 returncode, output = child_processes_manager.RunSubprocess( 439 cmd, env, timeout=SUBPROCESS_TIMEOUT) 440 441 if args.show_capture_stdout: 442 self.logger.info("Capture stdout: %s" % output) 443 444 if returncode == -1: 445 self.results.append(GroupedResult(GroupedResult.CaptureFailed, "", output, self.tests)) 446 return False 447 elif returncode == -2: 448 self.results.append(GroupedResult(GroupedResult.TimedOut, "", "", self.tests)) 449 return False 450 451 with open(results_file) as f: 452 test_results = json.load(f) 453 os.unlink(results_file) 454 for test in self.tests: 455 test_result = test_results['tests'][test.full_test_name] 456 if test_result['actual'] == 'SKIP': 457 test.skipped_by_suite = True 458 459 return True 460 461 def RemoveTestsThatDoNotProduceAppropriateTraceFiles(self): 462 continued_tests = [] 463 skipped_tests = [] 464 failed_to_trace_tests = [] 465 for test in self.tests: 466 if not test.CanRunReplay(self.trace_folder_path): 467 if test.skipped_by_suite: 468 skipped_tests.append(test) 469 else: 470 failed_to_trace_tests.append(test) 471 else: 472 continued_tests.append(test) 473 if len(skipped_tests) > 0: 474 self.results.append( 475 GroupedResult(GroupedResult.Skipped, "Skipping replay since test skipped by suite", 476 "", skipped_tests)) 477 if len(failed_to_trace_tests) > 0: 478 self.results.append( 479 GroupedResult(GroupedResult.FailedToTrace, 480 "Test not skipped but failed to produce trace files", "", 481 failed_to_trace_tests)) 482 483 return continued_tests 484 485 def BuildReplay(self, replay_build_dir, composite_file_id, tests, child_processes_manager): 486 # write gni file that holds all the traces files in a list 487 self.CreateTestNamesFile(composite_file_id, tests) 488 489 gn_args = [('angle_build_capture_replay_tests', 'true'), 490 ('angle_capture_replay_test_trace_dir', '"%s"' % self.trace_dir), 491 ('angle_capture_replay_composite_file_id', str(composite_file_id))] 492 returncode, output = child_processes_manager.RunGNGen(replay_build_dir, True, gn_args) 493 if returncode != 0: 494 self.logger.warning('GN failure output: %s' % output) 495 self.results.append( 496 GroupedResult(GroupedResult.CompileFailed, "Build replay failed at gn generation", 497 output, tests)) 498 return False 499 returncode, output = child_processes_manager.RunAutoNinja(replay_build_dir, REPLAY_BINARY, 500 True) 501 if returncode != 0: 502 self.logger.warning('Ninja failure output: %s' % output) 503 self.results.append( 504 GroupedResult(GroupedResult.CompileFailed, "Build replay failed at ninja", output, 505 tests)) 506 return False 507 return True 508 509 def RunReplay(self, args, replay_build_dir, replay_exe_path, child_processes_manager, tests): 510 extra_env = {} 511 if args.expose_nonconformant_features: 512 extra_env[ 513 'ANGLE_FEATURE_OVERRIDES_ENABLED'] = 'exposeNonConformantExtensionsAndVersions' 514 515 env = {**os.environ.copy(), **extra_env} 516 517 run_cmd = GetRunCommand(self.args, replay_exe_path) 518 self.logger.info('%s %s' % (_FormatEnv(extra_env), ' '.join(run_cmd))) 519 520 for test in tests: 521 self.UnlinkContextStateJsonFilesIfPresent(replay_build_dir, test.GetLabel()) 522 523 returncode, output = child_processes_manager.RunSubprocess( 524 run_cmd, env, timeout=SUBPROCESS_TIMEOUT) 525 if returncode == -1: 526 cmd = replay_exe_path 527 self.results.append( 528 GroupedResult(GroupedResult.ReplayFailed, "Replay run failed (%s)" % cmd, output, 529 tests)) 530 return 531 elif returncode == -2: 532 self.results.append( 533 GroupedResult(GroupedResult.TimedOut, "Replay run timed out", output, tests)) 534 return 535 536 if args.show_replay_stdout: 537 self.logger.info("Replay stdout: %s" % output) 538 539 output_lines = output.splitlines() 540 passes = [] 541 fails = [] 542 count = 0 543 for output_line in output_lines: 544 words = output_line.split(" ") 545 if len(words) == 3 and words[0] == RESULT_TAG: 546 test_name = self.FindTestByLabel(words[1]) 547 result = int(words[2]) 548 if result == 0: 549 passes.append(test_name) 550 elif result == REPLAY_INITIALIZATION_FAILURE: 551 fails.append(test_name) 552 self.logger.info("Initialization failure: %s" % test_name) 553 elif result == REPLAY_SERIALIZATION_FAILURE: 554 fails.append(test_name) 555 self.logger.info("Context comparison failed: %s" % test_name) 556 self.PrintContextDiff(replay_build_dir, words[1]) 557 else: 558 fails.append(test_name) 559 self.logger.error("Unknown test result code: %s -> %d" % (test_name, result)) 560 count += 1 561 562 if len(passes) > 0: 563 self.results.append(GroupedResult(GroupedResult.Passed, "", output, passes)) 564 if len(fails) > 0: 565 self.results.append(GroupedResult(GroupedResult.Failed, "", output, fails)) 566 567 def UnlinkContextStateJsonFilesIfPresent(self, replay_build_dir, test_name): 568 frame = 1 569 while True: 570 capture_file = "{}/{}_ContextCaptured{}.json".format(replay_build_dir, test_name, 571 frame) 572 replay_file = "{}/{}_ContextReplayed{}.json".format(replay_build_dir, test_name, frame) 573 if os.path.exists(capture_file): 574 os.unlink(capture_file) 575 if os.path.exists(replay_file): 576 os.unlink(replay_file) 577 578 if frame > self.CAPTURE_FRAME_END: 579 break 580 frame = frame + 1 581 582 def PrintContextDiff(self, replay_build_dir, test_name): 583 frame = 1 584 found = False 585 while True: 586 capture_file = "{}/{}_ContextCaptured{}.json".format(replay_build_dir, test_name, 587 frame) 588 replay_file = "{}/{}_ContextReplayed{}.json".format(replay_build_dir, test_name, frame) 589 if os.path.exists(capture_file) and os.path.exists(replay_file): 590 found = True 591 captured_context = open(capture_file, "r").readlines() 592 replayed_context = open(replay_file, "r").readlines() 593 for line in difflib.unified_diff( 594 captured_context, replayed_context, fromfile=capture_file, 595 tofile=replay_file): 596 print(line, end="") 597 else: 598 if frame > self.CAPTURE_FRAME_END: 599 break 600 frame = frame + 1 601 if not found: 602 self.logger.error("Could not find serialization diff files for %s" % test_name) 603 604 def FindTestByLabel(self, label): 605 for test in self.tests: 606 if test.GetLabel() == label: 607 return test 608 return None 609 610 def AddTest(self, test): 611 assert len(self.tests) <= self.args.batch_count 612 test.index = len(self.tests) 613 self.tests.append(test) 614 615 def CreateTestNamesFile(self, composite_file_id, tests): 616 data = {'traces': [test.GetLabel() for test in tests]} 617 names_path = os.path.join(self.trace_folder_path, 'test_names_%d.json' % composite_file_id) 618 with open(names_path, 'w') as f: 619 f.write(json.dumps(data)) 620 621 def __str__(self): 622 repr_str = "TestBatch:\n" 623 for test in self.tests: 624 repr_str += ("\t" + str(test) + "\n") 625 return repr_str 626 627 def __getitem__(self, index): 628 assert index < len(self.tests) 629 return self.tests[index] 630 631 def __iter__(self): 632 return iter(self.tests) 633 634 def GetResults(self): 635 return TestBatchResult(self.batch_index, self.results, self.args.verbose) 636 637 638class TestExpectation(): 639 # tests that must not be run as list 640 skipped_for_capture_tests = {} 641 skipped_for_capture_tests_re = {} 642 643 # test expectations for tests that do not pass 644 non_pass_results = {} 645 646 # tests that must run in a one-test batch 647 run_single = {} 648 run_single_re = {} 649 650 flaky_tests = [] 651 652 non_pass_re = {} 653 654 result_map = { 655 "FAIL": GroupedResult.Failed, 656 "TIMEOUT": GroupedResult.TimedOut, 657 "COMPILE_FAIL": GroupedResult.CompileFailed, 658 "NOT_RUN": GroupedResult.Skipped, 659 "SKIP_FOR_CAPTURE": GroupedResult.Skipped, 660 "PASS": GroupedResult.Passed, 661 } 662 663 def __init__(self, args): 664 expected_results_filename = "capture_replay_expectations.txt" 665 expected_results_path = os.path.join(REPLAY_SAMPLE_FOLDER, expected_results_filename) 666 self._asan = args.asan 667 with open(expected_results_path, "rt") as f: 668 for line in f: 669 l = line.strip() 670 if l != "" and not l.startswith("#"): 671 self.ReadOneExpectation(l, args.debug) 672 673 def _CheckTagsWithConfig(self, tags, config_tags): 674 for tag in tags: 675 if tag not in config_tags: 676 return False 677 return True 678 679 def ReadOneExpectation(self, line, is_debug): 680 (testpattern, result) = line.split('=') 681 (test_info_string, test_name_string) = testpattern.split(':') 682 test_name = test_name_string.strip() 683 test_info = test_info_string.strip().split() 684 result_stripped = result.strip() 685 686 tags = [] 687 if len(test_info) > 1: 688 tags = test_info[1:] 689 690 config_tags = [GetPlatformForSkip()] 691 if self._asan: 692 config_tags += ['ASAN'] 693 if is_debug: 694 config_tags += ['DEBUG'] 695 696 if self._CheckTagsWithConfig(tags, config_tags): 697 test_name_regex = re.compile('^' + test_name.replace('*', '.*') + '$') 698 if result_stripped == 'COMPILE_FAIL': 699 self.run_single[test_name] = self.result_map[result_stripped] 700 self.run_single_re[test_name] = test_name_regex 701 if result_stripped == 'SKIP_FOR_CAPTURE' or result_stripped == 'TIMEOUT': 702 self.skipped_for_capture_tests[test_name] = self.result_map[result_stripped] 703 self.skipped_for_capture_tests_re[test_name] = test_name_regex 704 elif result_stripped == 'FLAKY': 705 self.flaky_tests.append(test_name_regex) 706 else: 707 self.non_pass_results[test_name] = self.result_map[result_stripped] 708 self.non_pass_re[test_name] = test_name_regex 709 710 def TestIsSkippedForCapture(self, test_name): 711 return any(p.match(test_name) for p in self.skipped_for_capture_tests_re.values()) 712 713 def TestNeedsToRunSingle(self, test_name): 714 if any(p.match(test_name) for p in self.run_single_re.values()): 715 return True 716 717 return self.TestIsSkippedForCapture(test_name) 718 719 def Filter(self, test_list, run_all_tests): 720 result = {} 721 for t in test_list: 722 for key in self.non_pass_results.keys(): 723 if self.non_pass_re[key].match(t) is not None: 724 result[t] = self.non_pass_results[key] 725 for key in self.run_single.keys(): 726 if self.run_single_re[key].match(t) is not None: 727 result[t] = self.run_single[key] 728 if run_all_tests: 729 for [key, r] in self.skipped_for_capture_tests.items(): 730 if self.skipped_for_capture_tests_re[key].match(t) is not None: 731 result[t] = r 732 return result 733 734 def IsFlaky(self, test_name): 735 for flaky in self.flaky_tests: 736 if flaky.match(test_name) is not None: 737 return True 738 return False 739 740 741def ClearFolderContent(path): 742 all_files = [] 743 for f in os.listdir(path): 744 if os.path.isfile(os.path.join(path, f)): 745 os.remove(os.path.join(path, f)) 746 747def SetCWDToAngleFolder(): 748 cwd = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) 749 os.chdir(cwd) 750 return cwd 751 752 753def CleanupAfterReplay(replay_build_dir, tests): 754 # Remove files that have test labels in the file name, .e.g: 755 # ClearTest_ClearIsClamped_ES2_Vulkan_SwiftShader.dll.pdb 756 test_labels = [test.GetLabel() for test in tests] 757 for build_file in os.listdir(replay_build_dir): 758 if any(label in build_file for label in test_labels): 759 os.unlink(os.path.join(replay_build_dir, build_file)) 760 761 762def RunTests(args, worker_id, job_queue, result_list, message_queue, logger, ninja_lock): 763 replay_build_dir = os.path.join(args.out_dir, 'Replay%d' % worker_id) 764 replay_exec_path = os.path.join(replay_build_dir, REPLAY_BINARY) 765 766 child_processes_manager = ChildProcessesManager(args, logger, ninja_lock) 767 # used to differentiate between multiple composite files when there are multiple test batchs 768 # running on the same worker and --deleted_trace is set to False 769 composite_file_id = 1 770 while not job_queue.empty(): 771 try: 772 test_batch = job_queue.get() 773 logger.info('Starting {} ({} tests) on worker {}. Unstarted jobs: {}'.format( 774 BatchName(test_batch), len(test_batch.tests), worker_id, job_queue.qsize())) 775 776 test_batch.SetWorkerId(worker_id) 777 778 success = test_batch.RunWithCapture(args, child_processes_manager) 779 if not success: 780 result_list.append(test_batch.GetResults()) 781 logger.error('Failed RunWithCapture: %s', str(test_batch.GetResults())) 782 continue 783 continued_tests = test_batch.RemoveTestsThatDoNotProduceAppropriateTraceFiles() 784 if len(continued_tests) == 0: 785 result_list.append(test_batch.GetResults()) 786 logger.info('No tests to replay: %s', str(test_batch.GetResults())) 787 continue 788 success = test_batch.BuildReplay(replay_build_dir, composite_file_id, continued_tests, 789 child_processes_manager) 790 if args.keep_temp_files: 791 composite_file_id += 1 792 if not success: 793 result_list.append(test_batch.GetResults()) 794 logger.error('Failed BuildReplay: %s', str(test_batch.GetResults())) 795 continue 796 test_batch.RunReplay(args, replay_build_dir, replay_exec_path, child_processes_manager, 797 continued_tests) 798 result_list.append(test_batch.GetResults()) 799 if not args.keep_temp_files: 800 CleanupAfterReplay(replay_build_dir, continued_tests) 801 logger.info('Finished RunReplay: %s', str(test_batch.GetResults())) 802 except KeyboardInterrupt: 803 child_processes_manager.KillAll() 804 raise 805 except queue.Empty: 806 child_processes_manager.KillAll() 807 break 808 except Exception as e: 809 logger.error('RunTestsException: %s\n%s' % (repr(e), traceback.format_exc())) 810 child_processes_manager.KillAll() 811 pass 812 message_queue.put(child_processes_manager.runtimes) 813 child_processes_manager.KillAll() 814 815 816def SafeDeleteFolder(folder_name): 817 while os.path.isdir(folder_name): 818 try: 819 shutil.rmtree(folder_name) 820 except KeyboardInterrupt: 821 raise 822 except PermissionError: 823 pass 824 825 826def DeleteReplayBuildFolders(folder_num, replay_build_dir, trace_folder): 827 for i in range(folder_num): 828 folder_name = replay_build_dir + str(i) 829 if os.path.isdir(folder_name): 830 SafeDeleteFolder(folder_name) 831 832 833def CreateTraceFolders(folder_num): 834 for i in range(folder_num): 835 folder_name = TRACE_FOLDER + str(i) 836 folder_path = os.path.join(REPLAY_SAMPLE_FOLDER, folder_name) 837 if os.path.isdir(folder_path): 838 shutil.rmtree(folder_path) 839 os.makedirs(folder_path) 840 841 842def DeleteTraceFolders(folder_num): 843 for i in range(folder_num): 844 folder_name = TRACE_FOLDER + str(i) 845 folder_path = os.path.join(REPLAY_SAMPLE_FOLDER, folder_name) 846 if os.path.isdir(folder_path): 847 SafeDeleteFolder(folder_path) 848 849 850def GetPlatformForSkip(): 851 # yapf: disable 852 # we want each pair on one line 853 platform_map = { 'win32' : 'WIN', 854 'linux' : 'LINUX' } 855 # yapf: enable 856 return platform_map.get(sys.platform, 'UNKNOWN') 857 858 859def main(args): 860 logger = multiprocessing.log_to_stderr() 861 logger.setLevel(level=args.log.upper()) 862 863 is_bot = getpass.getuser() == 'chrome-bot' 864 865 if is_bot: 866 # bots need different re-client auth settings than developers b/319246651 867 os.environ["RBE_use_gce_credentials"] = "true" 868 os.environ["RBE_use_application_default_credentials"] = "false" 869 os.environ["RBE_automatic_auth"] = "false" 870 os.environ["RBE_experimental_credentials_helper"] = "" 871 os.environ["RBE_experimental_credentials_helper_args"] = "" 872 873 if sys.platform == 'linux' and is_bot: 874 logger.warning('Test is currently a no-op https://anglebug.com/6085') 875 return EXIT_SUCCESS 876 877 ninja_lock = multiprocessing.Semaphore(args.max_ninja_jobs) 878 child_processes_manager = ChildProcessesManager(args, logger, ninja_lock) 879 try: 880 start_time = time.time() 881 # set the number of workers to be cpu_count - 1 (since the main process already takes up a 882 # CPU core). Whenever a worker is available, it grabs the next job from the job queue and 883 # runs it. The worker closes down when there is no more job. 884 worker_count = min(multiprocessing.cpu_count() - 1, args.max_jobs) 885 cwd = SetCWDToAngleFolder() 886 887 CreateTraceFolders(worker_count) 888 capture_build_dir = os.path.normpath(r'%s/Capture' % args.out_dir) 889 returncode, output = child_processes_manager.RunGNGen(capture_build_dir, False) 890 if returncode != 0: 891 logger.error(output) 892 child_processes_manager.KillAll() 893 return EXIT_FAILURE 894 # run ninja to build all tests 895 returncode, output = child_processes_manager.RunAutoNinja(capture_build_dir, 896 args.test_suite, False) 897 if returncode != 0: 898 logger.error(output) 899 child_processes_manager.KillAll() 900 return EXIT_FAILURE 901 # get a list of tests 902 test_path = os.path.join(capture_build_dir, args.test_suite) 903 test_list = GetTestsListForFilter(args, test_path, args.filter, logger) 904 test_expectation = TestExpectation(args) 905 test_names = ParseTestNamesFromTestList(test_list, test_expectation, 906 args.also_run_skipped_for_capture_tests, logger) 907 test_expectation_for_list = test_expectation.Filter( 908 test_names, args.also_run_skipped_for_capture_tests) 909 # objects created by manager can be shared by multiple processes. We use it to create 910 # collections that are shared by multiple processes such as job queue or result list. 911 manager = multiprocessing.Manager() 912 job_queue = manager.Queue() 913 test_batch_num = 0 914 915 num_tests = len(test_names) 916 test_index = 0 917 918 # Put the tests into batches and these into the job queue; jobs that areexpected to crash, 919 # timeout, or fail compilation will be run in batches of size one, because a crash or 920 # failing to compile brings down the whole batch, so that we would give false negatives if 921 # such a batch contains jobs that would otherwise poss or fail differently. 922 batch_index = 0 923 while test_index < num_tests: 924 batch = TestBatch(args, logger, batch_index) 925 batch_index += 1 926 927 while test_index < num_tests and len(batch.tests) < args.batch_count: 928 test_name = test_names[test_index] 929 test_obj = Test(test_name) 930 931 if test_expectation.TestNeedsToRunSingle(test_name): 932 single_batch = TestBatch(args, logger, batch_index) 933 batch_index += 1 934 single_batch.AddTest(test_obj) 935 job_queue.put(single_batch) 936 test_batch_num += 1 937 else: 938 batch.AddTest(test_obj) 939 940 test_index += 1 941 942 if len(batch.tests) > 0: 943 job_queue.put(batch) 944 test_batch_num += 1 945 946 unexpected_count = {} 947 unexpected_test_results = {} 948 949 for type in GroupedResult.ResultTypes: 950 unexpected_count[type] = 0 951 unexpected_test_results[type] = [] 952 953 # result list is created by manager and can be shared by multiple processes. Each 954 # subprocess populates the result list with the results of its test runs. After all 955 # subprocesses finish, the main process processes the results in the result list. 956 # An item in the result list is a tuple with 3 values (testname, result, output). 957 # The "result" can take 3 values "Passed", "Failed", "Skipped". The output is the 958 # stdout and the stderr of the test appended together. 959 result_list = manager.list() 960 message_queue = manager.Queue() 961 # so that we do not spawn more processes than we actually need 962 worker_count = min(worker_count, test_batch_num) 963 # spawning and starting up workers 964 for worker_id in range(worker_count): 965 proc = multiprocessing.Process( 966 target=RunTests, 967 args=(args, worker_id, job_queue, result_list, message_queue, logger, ninja_lock)) 968 child_processes_manager.AddWorker(proc) 969 proc.start() 970 971 # print out periodic status messages 972 while child_processes_manager.IsAnyWorkerAlive(): 973 logger.info('%d workers running, %d jobs left.' % 974 (child_processes_manager.GetRemainingWorkers(), (job_queue.qsize()))) 975 # If only a few tests are run it is likely that the workers are finished before 976 # the STATUS_MESSAGE_PERIOD has passed, and the tests script sits idle for the 977 # reminder of the wait time. Therefore, limit waiting by the number of 978 # unfinished jobs. 979 unfinished_jobs = job_queue.qsize() + child_processes_manager.GetRemainingWorkers() 980 time.sleep(min(STATUS_MESSAGE_PERIOD, unfinished_jobs)) 981 982 child_processes_manager.JoinWorkers() 983 end_time = time.time() 984 985 summed_runtimes = child_processes_manager.runtimes 986 while not message_queue.empty(): 987 runtimes = message_queue.get() 988 for k, v in runtimes.items(): 989 summed_runtimes.setdefault(k, 0.0) 990 summed_runtimes[k] += v 991 992 # print out results 993 logger.info('') 994 logger.info('Results:') 995 996 flaky_results = [] 997 998 for test_batch in result_list: 999 test_batch_result = test_batch.results 1000 logger.debug(str(test_batch_result)) 1001 1002 for real_result, test_list in test_batch_result.items(): 1003 for test in test_list: 1004 if test_expectation.IsFlaky(test): 1005 flaky_results.append('{} ({})'.format(test, real_result)) 1006 continue 1007 1008 expected_result = test_expectation_for_list.get(test, GroupedResult.Passed) 1009 1010 if real_result not in (GroupedResult.Passed, expected_result): 1011 unexpected_count[real_result] += 1 1012 unexpected_test_results[real_result].append('!= {}: {} {}'.format( 1013 expected_result, BatchName(test_batch), test)) 1014 1015 logger.info('') 1016 logger.info('Elapsed time: %.2lf seconds' % (end_time - start_time)) 1017 logger.info('') 1018 logger.info('Runtimes by process:\n%s' % 1019 '\n'.join('%s: %.2lf seconds' % (k, v) for (k, v) in summed_runtimes.items())) 1020 1021 if len(flaky_results): 1022 logger.info("Test(s) marked as flaky (not considered a failure):") 1023 for line in flaky_results: 1024 logger.info(" {}".format(line)) 1025 logger.info("") 1026 1027 retval = EXIT_SUCCESS 1028 1029 unexpected_test_results_count = 0 1030 for result, count in unexpected_count.items(): 1031 if result != GroupedResult.Skipped: # Suite skipping tests is ok 1032 unexpected_test_results_count += count 1033 1034 if unexpected_test_results_count > 0: 1035 retval = EXIT_FAILURE 1036 logger.info('') 1037 logger.info('Failure: Obtained {} results that differ from expectation:'.format( 1038 unexpected_test_results_count)) 1039 logger.info('') 1040 for result, count in unexpected_count.items(): 1041 if count > 0 and result != GroupedResult.Skipped: 1042 logger.info("Unexpected '{}' ({}):".format(result, count)) 1043 for test_result in unexpected_test_results[result]: 1044 logger.info(' {}'.format(test_result)) 1045 logger.info('') 1046 1047 logger.info('') 1048 1049 # delete generated folders if --keep-temp-files flag is set to false 1050 if args.purge: 1051 DeleteTraceFolders(worker_count) 1052 if os.path.isdir(args.out_dir): 1053 SafeDeleteFolder(args.out_dir) 1054 1055 # Try hard to ensure output is finished before ending the test. 1056 logging.shutdown() 1057 sys.stdout.flush() 1058 time.sleep(2.0) 1059 return retval 1060 1061 except KeyboardInterrupt: 1062 child_processes_manager.KillAll() 1063 return EXIT_FAILURE 1064 1065 1066if __name__ == '__main__': 1067 parser = argparse.ArgumentParser() 1068 parser.add_argument( 1069 '--out-dir', 1070 default=DEFAULT_OUT_DIR, 1071 help='Where to build ANGLE for capture and replay. Relative to the ANGLE folder. Default is "%s".' 1072 % DEFAULT_OUT_DIR) 1073 parser.add_argument( 1074 '-f', 1075 '--filter', 1076 '--gtest_filter', 1077 default=DEFAULT_FILTER, 1078 help='Same as GoogleTest\'s filter argument. Default is "%s".' % DEFAULT_FILTER) 1079 parser.add_argument( 1080 '--test-suite', 1081 default=DEFAULT_TEST_SUITE, 1082 help='Test suite binary to execute. Default is "%s".' % DEFAULT_TEST_SUITE) 1083 parser.add_argument( 1084 '--batch-count', 1085 default=DEFAULT_BATCH_COUNT, 1086 type=int, 1087 help='Number of tests in a batch. Default is %d.' % DEFAULT_BATCH_COUNT) 1088 parser.add_argument( 1089 '--keep-temp-files', 1090 action='store_true', 1091 help='Whether to keep the temp files and folders. Off by default') 1092 parser.add_argument('--purge', help='Purge all build directories on exit.') 1093 parser.add_argument( 1094 '--use-reclient', 1095 default=False, 1096 action='store_true', 1097 help='Set use_remoteexec=true in args.gn.') 1098 parser.add_argument( 1099 '--output-to-file', 1100 action='store_true', 1101 help='Whether to write output to a result file. Off by default') 1102 parser.add_argument( 1103 '--result-file', 1104 default=DEFAULT_RESULT_FILE, 1105 help='Name of the result file in the capture_replay_tests folder. Default is "%s".' % 1106 DEFAULT_RESULT_FILE) 1107 parser.add_argument('-v', '--verbose', action='store_true', help='Shows full test output.') 1108 parser.add_argument( 1109 '-l', 1110 '--log', 1111 default=DEFAULT_LOG_LEVEL, 1112 help='Controls the logging level. Default is "%s".' % DEFAULT_LOG_LEVEL) 1113 parser.add_argument( 1114 '-j', 1115 '--max-jobs', 1116 default=DEFAULT_MAX_JOBS, 1117 type=int, 1118 help='Maximum number of test processes. Default is %d.' % DEFAULT_MAX_JOBS) 1119 parser.add_argument( 1120 '-M', 1121 '--mec', 1122 default=0, 1123 type=int, 1124 help='Enable mid execution capture starting at specified frame, (default: 0 = normal capture)' 1125 ) 1126 parser.add_argument( 1127 '-a', 1128 '--also-run-skipped-for-capture-tests', 1129 action='store_true', 1130 help='Also run tests that are disabled in the expectations by SKIP_FOR_CAPTURE') 1131 parser.add_argument( 1132 '--max-ninja-jobs', 1133 type=int, 1134 default=DEFAULT_MAX_NINJA_JOBS, 1135 help='Maximum number of concurrent ninja jobs to run at once.') 1136 parser.add_argument('--xvfb', action='store_true', help='Run with xvfb.') 1137 parser.add_argument('--asan', action='store_true', help='Build with ASAN.') 1138 parser.add_argument( 1139 '-E', 1140 '--expose-nonconformant-features', 1141 action='store_true', 1142 help='Expose non-conformant features to advertise GLES 3.2') 1143 parser.add_argument( 1144 '--show-capture-stdout', action='store_true', help='Print test stdout during capture.') 1145 parser.add_argument( 1146 '--show-replay-stdout', action='store_true', help='Print test stdout during replay.') 1147 parser.add_argument('--debug', action='store_true', help='Debug builds (default is Release).') 1148 args = parser.parse_args() 1149 if args.debug and (args.out_dir == DEFAULT_OUT_DIR): 1150 args.out_dir = args.out_dir + "Debug" 1151 1152 if sys.platform == "win32": 1153 args.test_suite += ".exe" 1154 if args.output_to_file: 1155 logging.basicConfig(level=args.log.upper(), filename=args.result_file) 1156 else: 1157 logging.basicConfig(level=args.log.upper()) 1158 1159 sys.exit(main(args)) 1160