• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/env python3
2#
3# Copyright 2020 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7'''
8Script that re-captures the traces in the restricted trace folder. We can
9use this to update traces without needing to re-run the app on a device.
10'''
11
12import argparse
13import fnmatch
14import json
15import logging
16import os
17import shutil
18import stat
19import subprocess
20import sys
21
22from gen_restricted_traces import read_json as read_json
23
24DEFAULT_TEST_SUITE = 'angle_perftests'
25DEFAULT_TEST_JSON = 'restricted_traces.json'
26DEFAULT_LOG_LEVEL = 'info'
27DEFAULT_BACKUP_FOLDER = 'retrace-backups'
28
29EXIT_SUCCESS = 0
30EXIT_FAILURE = 1
31
32
33def get_trace_json_path(trace):
34    return os.path.join(get_script_dir(), trace, f'{trace}.json')
35
36
37def load_trace_json(trace):
38    json_file_name = get_trace_json_path(trace)
39    return read_json(json_file_name)
40
41
42def get_context(trace):
43    """Returns the trace context number."""
44    json_data = load_trace_json(trace)
45    return str(json_data['WindowSurfaceContextID'])
46
47
48def get_script_dir():
49    return os.path.dirname(sys.argv[0])
50
51
52def context_header(trace, trace_path):
53    context_id = get_context(trace)
54    header = '%s_context%s.h' % (trace, context_id)
55    return os.path.join(trace_path, header)
56
57
58def src_trace_path(trace):
59    return os.path.join(get_script_dir(), trace)
60
61
62def get_num_frames(json_data):
63    metadata = json_data['TraceMetadata']
64    return metadata['FrameEnd'] - metadata['FrameStart'] + 1
65
66
67def path_contains_header(path):
68    for file in os.listdir(path):
69        if fnmatch.fnmatch(file, '*.h'):
70            return True
71    return False
72
73
74def chmod_directory(directory, perm):
75    assert os.path.isdir(directory)
76    for file in os.listdir(directory):
77        fn = os.path.join(directory, file)
78        os.chmod(fn, perm)
79
80
81def ensure_rmdir(directory):
82    if os.path.isdir(directory):
83        chmod_directory(directory, stat.S_IWRITE)
84        shutil.rmtree(directory)
85
86
87def copy_trace_folder(old_path, new_path):
88    logging.info('%s -> %s' % (old_path, new_path))
89    ensure_rmdir(new_path)
90    shutil.copytree(old_path, new_path)
91
92
93def backup_traces(args, traces):
94    for trace in fnmatch.filter(traces, args.traces):
95        trace_path = src_trace_path(trace)
96        trace_backup_path = os.path.join(args.out_path, trace)
97        copy_trace_folder(trace_path, trace_backup_path)
98
99
100def restore_traces(args, traces):
101    for trace in fnmatch.filter(traces, args.traces):
102        trace_path = src_trace_path(trace)
103        trace_backup_path = os.path.join(args.out_path, trace)
104        if not os.path.isdir(trace_backup_path):
105            logging.error('Trace folder not found at %s' % trace_backup_path)
106        else:
107            copy_trace_folder(trace_backup_path, trace_path)
108
109
110def run_autoninja(args):
111    autoninja_binary = 'autoninja'
112    if os.name == 'nt':
113        autoninja_binary += '.bat'
114
115    autoninja_args = [autoninja_binary, '-C', args.gn_path, args.test_suite]
116    logging.debug('Calling %s' % ' '.join(autoninja_args))
117    subprocess.check_call(autoninja_args)
118
119
120def run_test_suite(args, trace, max_steps, additional_args, additional_env):
121    trace_binary = os.path.join(args.gn_path, args.test_suite)
122    if os.name == 'nt':
123        trace_binary += '.exe'
124
125    renderer = 'vulkan' if args.no_swiftshader else 'vulkan_swiftshader'
126    trace_filter = '--gtest_filter=TracePerfTest.Run/%s_%s' % (renderer, trace)
127    run_args = [
128        trace_binary,
129        trace_filter,
130        '--max-steps-performed',
131        str(max_steps),
132    ] + additional_args
133    if not args.no_swiftshader:
134        run_args += ['--enable-all-trace-tests']
135
136    env = {**os.environ.copy(), **additional_env}
137    env_string = ' '.join(['%s=%s' % item for item in additional_env.items()])
138    if env_string:
139        env_string += ' '
140
141    logging.info('%s%s' % (env_string, ' '.join(run_args)))
142    subprocess.check_call(run_args, env=env)
143
144
145def upgrade_traces(args, traces):
146    run_autoninja(args)
147
148    failures = []
149
150    for trace in fnmatch.filter(traces, args.traces):
151        logging.debug('Tracing %s' % trace)
152
153        trace_path = os.path.abspath(os.path.join(args.out_path, trace))
154        if not os.path.isdir(trace_path):
155            os.makedirs(trace_path)
156        elif args.no_overwrite and path_contains_header(trace_path):
157            logging.info('Skipping "%s" because the out folder already exists' % trace)
158            continue
159
160        json_data = load_trace_json(trace)
161        num_frames = get_num_frames(json_data)
162
163        metadata = json_data['TraceMetadata']
164        logging.debug('Read metadata: %s' % str(metadata))
165
166        max_steps = min(args.limit, num_frames) if args.limit else num_frames
167
168        # We start tracing from frame 2. --retrace-mode issues a Swap() after Setup() so we can
169        # accurately re-trace the MEC.
170        additional_env = {
171            'ANGLE_CAPTURE_LABEL': trace,
172            'ANGLE_CAPTURE_OUT_DIR': trace_path,
173            'ANGLE_CAPTURE_FRAME_START': '2',
174            'ANGLE_CAPTURE_FRAME_END': str(max_steps + 1),
175        }
176        if args.validation:
177            additional_env['ANGLE_CAPTURE_VALIDATION'] = '1'
178            # Also turn on shader output init to ensure we have no undefined values.
179            # This feature is also enabled in replay when using --validation.
180            additional_env[
181                'ANGLE_FEATURE_OVERRIDES_ENABLED'] = 'allocateNonZeroMemory:forceInitShaderVariables'
182        if args.validation_expr:
183            additional_env['ANGLE_CAPTURE_VALIDATION_EXPR'] = args.validation_expr
184        if args.trim:
185            additional_env['ANGLE_CAPTURE_TRIM_ENABLED'] = '1'
186        if args.no_trim:
187            additional_env['ANGLE_CAPTURE_TRIM_ENABLED'] = '0'
188
189        additional_args = ['--retrace-mode']
190
191        try:
192            run_test_suite(args, trace, max_steps, additional_args, additional_env)
193
194            json_file = get_trace_json_path(trace)
195            if not os.path.exists(json_file):
196                logging.error(
197                    f'There was a problem tracing "{trace}", could not find json file: {json_file}'
198                )
199                failures += [trace]
200        except:
201            logging.exception('There was an exception running "%s":' % trace)
202            failures += [trace]
203
204    if failures:
205        print('The following traces failed to upgrade:\n')
206        print('\n'.join(['  ' + trace for trace in failures]))
207        return EXIT_FAILURE
208
209    return EXIT_SUCCESS
210
211
212def validate_traces(args, traces):
213    restore_traces(args, traces)
214    run_autoninja(args)
215
216    additional_args = ['--validation']
217    additional_env = {
218        'ANGLE_FEATURE_OVERRIDES_ENABLED': 'allocateNonZeroMemory:forceInitShaderVariables'
219    }
220
221    failures = []
222
223    for trace in fnmatch.filter(traces, args.traces):
224        json_data = load_trace_json(trace)
225        num_frames = get_num_frames(json_data)
226        max_steps = min(args.limit, num_frames) if args.limit else num_frames
227        try:
228            run_test_suite(args, trace, max_steps, additional_args, additional_env)
229        except:
230            logging.error('There was a failure running "%s".' % trace)
231            failures += [trace]
232
233    if failures:
234        print('The following traces failed to validate:\n')
235        print('\n'.join(['  ' + trace for trace in failures]))
236        return EXIT_FAILURE
237
238    return EXIT_SUCCESS
239
240
241def main():
242    parser = argparse.ArgumentParser()
243    parser.add_argument('-l', '--log', help='Logging level.', default=DEFAULT_LOG_LEVEL)
244    parser.add_argument(
245        '--test-suite',
246        help='Test Suite. Default is %s' % DEFAULT_TEST_SUITE,
247        default=DEFAULT_TEST_SUITE)
248    parser.add_argument(
249        '--no-swiftshader',
250        help='Trace against native Vulkan.',
251        action='store_true',
252        default=False)
253
254    subparsers = parser.add_subparsers(dest='command', required=True, help='Command to run.')
255
256    backup_parser = subparsers.add_parser(
257        'backup', help='Copies trace contents into a saved folder.')
258    backup_parser.add_argument(
259        'traces', help='Traces to back up. Supports fnmatch expressions.', default='*')
260    backup_parser.add_argument(
261        '-o',
262        '--out-path',
263        '--backup-path',
264        help='Destination folder. Default is "%s".' % DEFAULT_BACKUP_FOLDER,
265        default=DEFAULT_BACKUP_FOLDER)
266
267    restore_parser = subparsers.add_parser(
268        'restore', help='Copies traces from a saved folder to the trace folder.')
269    restore_parser.add_argument(
270        '-o',
271        '--out-path',
272        '--backup-path',
273        help='Path the traces were saved. Default is "%s".' % DEFAULT_BACKUP_FOLDER,
274        default=DEFAULT_BACKUP_FOLDER)
275    restore_parser.add_argument(
276        'traces', help='Traces to restore. Supports fnmatch expressions.', default='*')
277
278    upgrade_parser = subparsers.add_parser(
279        'upgrade', help='Re-trace existing traces, upgrading the format.')
280    upgrade_parser.add_argument('gn_path', help='GN build path')
281    upgrade_parser.add_argument('out_path', help='Output directory')
282    upgrade_parser.add_argument(
283        '-f', '--traces', '--filter', help='Trace filter. Defaults to all.', default='*')
284    upgrade_parser.add_argument(
285        '-n',
286        '--no-overwrite',
287        help='Skip traces which already exist in the out directory.',
288        action='store_true')
289    upgrade_parser.add_argument(
290        '--validation', help='Enable state serialization validation calls.', action='store_true')
291    upgrade_parser.add_argument(
292        '--validation-expr',
293        help='Validation expression, used to add more validation checkpoints.')
294    upgrade_parser.add_argument(
295        '--limit',
296        '--frame-limit',
297        type=int,
298        help='Limits the number of captured frames to produce a shorter trace than the original.')
299    upgrade_parser.add_argument(
300        '--trim', action='store_true', help='Enables trace trimming. Breaks replay validation.')
301    upgrade_parser.add_argument(
302        '--no-trim', action='store_true', help='Disables trace trimming. Useful for validation.')
303    upgrade_parser.set_defaults(trim=True)
304
305    validate_parser = subparsers.add_parser(
306        'validate', help='Runs the an updated test suite with validation enabled.')
307    validate_parser.add_argument('gn_path', help='GN build path')
308    validate_parser.add_argument('out_path', help='Path to the upgraded trace folder.')
309    validate_parser.add_argument(
310        'traces', help='Traces to validate. Supports fnmatch expressions.', default='*')
311    validate_parser.add_argument(
312        '--limit', '--frame-limit', type=int, help='Limits the number of tested frames.')
313
314    args, extra_flags = parser.parse_known_args()
315
316    logging.basicConfig(level=args.log.upper())
317
318    # Load trace names
319    with open(os.path.join(get_script_dir(), DEFAULT_TEST_JSON)) as f:
320        traces = json.loads(f.read())
321
322    traces = [trace.split(' ')[0] for trace in traces['traces']]
323
324    if args.command == 'backup':
325        return backup_traces(args, traces)
326    elif args.command == 'restore':
327        return restore_traces(args, traces)
328    elif args.command == 'upgrade':
329        return upgrade_traces(args, traces)
330    elif args.command == 'validate':
331        return validate_traces(args, traces)
332    else:
333        logging.fatal('Unknown command: %s' % args.command)
334        return EXIT_FAILURE
335
336
337if __name__ == '__main__':
338    sys.exit(main())
339