• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/env python3
2#
3# Copyright 2023 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7
8import argparse
9import contextlib
10import difflib
11import json
12import logging
13import os
14import pathlib
15import shutil
16import subprocess
17import sys
18import tempfile
19import time
20
21SCRIPT_DIR = str(pathlib.Path(__file__).resolve().parent)
22PY_UTILS = str(pathlib.Path(SCRIPT_DIR) / '..' / 'py_utils')
23if PY_UTILS not in sys.path:
24    os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS)
25import angle_test_util
26
27
28@contextlib.contextmanager
29def temporary_dir(prefix=''):
30    path = tempfile.mkdtemp(prefix=prefix)
31    try:
32        yield path
33    finally:
34        logging.info("Removing temporary directory: %s" % path)
35        shutil.rmtree(path)
36
37
38def file_content(path):
39    with open(path, 'rb') as f:
40        content = f.read()
41
42    if path.endswith('.json'):
43        info = json.loads(content)
44        info['TraceMetadata']['CaptureRevision'] = '<ignored>'
45        return json.dumps(info, indent=2).encode()
46
47    return content
48
49
50def diff_files(path, expected_path):
51    content = file_content(path)
52    expected_content = file_content(expected_path)
53    fn = os.path.basename(path)
54
55    if content == expected_content:
56        return False
57
58    if fn.endswith('.angledata'):
59        logging.error('Checks failed. Binary file contents mismatch: %s', fn)
60        return True
61
62    # Captured files are expected to have LF line endings.
63    # Note that git's EOL conversion for these files is disabled via .gitattributes
64    assert b'\r\n' not in content
65    assert b'\r\n' not in expected_content
66
67    diff = list(
68        difflib.unified_diff(
69            expected_content.decode().splitlines(),
70            content.decode().splitlines(),
71            fromfile=fn,
72            tofile=fn,
73        ))
74
75    logging.error('Checks failed. Found diff in %s:\n%s\n', fn, '\n'.join(diff))
76    return True
77
78
79def run_test(test_name, overwrite_expected):
80    with temporary_dir() as temp_dir:
81        exe = angle_test_util.ExecutablePathInCurrentDir('angle_end2end_tests')
82        test_args = ['--gtest_filter=%s' % test_name, '--angle-per-test-capture-label']
83        extra_env = {
84            'ANGLE_FEATURE_OVERRIDES_ENABLED': 'forceRobustResourceInit:forceInitShaderVariables',
85            'ANGLE_CAPTURE_ENABLED': '1',
86            'ANGLE_CAPTURE_FRAME_START': '2',
87            'ANGLE_CAPTURE_FRAME_END': '5',
88            'ANGLE_CAPTURE_OUT_DIR': temp_dir,
89            'ANGLE_CAPTURE_COMPRESSION': '0',
90        }
91        subprocess.check_call([exe] + test_args, env={**os.environ.copy(), **extra_env})
92        logging.info('Capture finished, comparing files')
93        files = sorted(fn for fn in os.listdir(temp_dir))
94        expected_dir = os.path.join(SCRIPT_DIR, 'expected')
95        expected_files = sorted(fn for fn in os.listdir(expected_dir) if not fn.startswith('.'))
96
97        if overwrite_expected:
98            for f in expected_files:
99                os.remove(os.path.join(expected_dir, f))
100            shutil.copytree(temp_dir, expected_dir, dirs_exist_ok=True)
101            return True
102
103        if files != expected_files:
104            logging.error(
105                'Checks failed. Capture produced a different set of files: %s\nDiff:\n%s\n', files,
106                '\n'.join(difflib.unified_diff(expected_files, files)))
107            return False
108
109        has_diffs = False
110        for fn in files:
111            has_diffs |= diff_files(os.path.join(temp_dir, fn), os.path.join(expected_dir, fn))
112
113        return not has_diffs
114
115
116def main():
117    parser = argparse.ArgumentParser()
118    parser.add_argument('--isolated-script-test-output', type=str)
119    parser.add_argument('--log', help='Logging level.', default='info')
120    parser.add_argument(
121        '--overwrite-expected', help='Overwrite contents of expected/', action='store_true')
122    args, extra_flags = parser.parse_known_args()
123
124    logging.basicConfig(level=args.log.upper())
125
126    test_name = 'CapturedTest.MultiFrame/ES3_Vulkan'
127    had_error = False
128    try:
129        if not run_test(test_name, args.overwrite_expected):
130            had_error = True
131            logging.error(
132                'Found capture diffs. If diffs are expected, build angle_end2end_tests and run '
133                '(cd out/<build>; ../../src/tests/capture_tests/capture_tests.py --overwrite-expected)'
134            )
135    except Exception as e:
136        logging.exception(e)
137        had_error = True
138
139    if args.isolated_script_test_output:
140        results = {
141            'tests': {
142                'capture_test': {}
143            },
144            'interrupted': False,
145            'seconds_since_epoch': time.time(),
146            'path_delimiter': '.',
147            'version': 3,
148            'num_failures_by_type': {
149                'FAIL': 0,
150                'PASS': 0,
151                'SKIP': 0,
152            },
153        }
154        result = 'FAIL' if had_error else 'PASS'
155        results['tests']['capture_test'][test_name] = {'expected': 'PASS', 'actual': result}
156        results['num_failures_by_type'][result] += 1
157
158        with open(args.isolated_script_test_output, 'w') as f:
159            f.write(json.dumps(results, indent=2))
160
161    return 1 if had_error else 0
162
163
164if __name__ == '__main__':
165    sys.exit(main())
166