• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2#
3# Copyright 2020 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# gen_restricted_traces.py:
8#   Generates integration code for the restricted trace tests.
9
10import getpass
11import glob
12import fnmatch
13import re
14import json
15import os
16import sys
17
18GNI_TEMPLATE = """\
19# GENERATED FILE - DO NOT EDIT.
20# Generated by {script_name} using data from {data_source_name}
21#
22# Copyright 2020 The ANGLE Project Authors. All rights reserved.
23# Use of this source code is governed by a BSD-style license that can be
24# found in the LICENSE file.
25#
26# A list of all restricted trace tests, paired with their context.
27# Can be consumed by tests/BUILD.gn.
28
29angle_restricted_traces = [
30{test_list}
31]
32"""
33
34HEADER_TEMPLATE = """\
35// GENERATED FILE - DO NOT EDIT.
36// Generated by {script_name} using data from {data_source_name}
37//
38// Copyright 2020 The ANGLE Project Authors. All rights reserved.
39// Use of this source code is governed by a BSD-style license that can be
40// found in the LICENSE file.
41//
42// {filename}: Types and enumerations for trace tests.
43
44#ifndef ANGLE_RESTRICTED_TRACES_AUTOGEN_H_
45#define ANGLE_RESTRICTED_TRACES_AUTOGEN_H_
46
47#include <cstdint>
48#include <vector>
49#include <KHR/khrplatform.h>
50#include <EGL/egl.h>
51
52#include "restricted_traces_export.h"
53
54namespace trace_angle
55{{
56using GenericProc = void (*)();
57using LoadProc    = GenericProc(KHRONOS_APIENTRY *)(const char *);
58ANGLE_TRACE_LOADER_EXPORT void LoadEGL(LoadProc loadProc);
59ANGLE_TRACE_LOADER_EXPORT void LoadGLES(LoadProc loadProc);
60
61static constexpr size_t kTraceInfoMaxNameLen = 128;
62
63static constexpr uint32_t kDefaultReplayContextClientMajorVersion = 3;
64static constexpr uint32_t kDefaultReplayContextClientMinorVersion = 1;
65static constexpr uint32_t kDefaultReplayDrawSurfaceColorSpace = EGL_COLORSPACE_LINEAR;
66
67struct TraceInfo
68{{
69    char name[kTraceInfoMaxNameLen];
70    uint32_t contextClientMajorVersion;
71    uint32_t contextClientMinorVersion;
72    uint32_t frameEnd;
73    uint32_t frameStart;
74    uint32_t drawSurfaceWidth;
75    uint32_t drawSurfaceHeight;
76    uint32_t drawSurfaceColorSpace;
77    uint32_t displayPlatformType;
78    uint32_t displayDeviceType;
79    int configRedBits;
80    int configBlueBits;
81    int configGreenBits;
82    int configAlphaBits;
83    int configDepthBits;
84    int configStencilBits;
85    bool isBinaryDataCompressed;
86    bool areClientArraysEnabled;
87    bool isBindGeneratesResourcesEnabled;
88    bool isWebGLCompatibilityEnabled;
89    bool isRobustResourceInitEnabled;
90}};
91
92ANGLE_TRACE_EXPORT const TraceInfo &GetTraceInfo(const char *traceName);
93}}  // namespace trace_angle
94
95#endif  // ANGLE_RESTRICTED_TRACES_AUTOGEN_H_
96"""
97
98SOURCE_TEMPLATE = """\
99// GENERATED FILE - DO NOT EDIT.
100// Generated by {script_name} using data from {data_source_name}
101//
102// Copyright 2020 The ANGLE Project Authors. All rights reserved.
103// Use of this source code is governed by a BSD-style license that can be
104// found in the LICENSE file.
105//
106// {filename}: Types and enumerations for trace tests.
107
108#include "{filename}.h"
109
110#include "common/PackedEnums.h"
111#include "common/system_utils.h"
112
113{trace_includes}
114
115namespace trace_angle
116{{
117namespace
118{{
119constexpr size_t kNumTraces = {num_traces};
120struct TracePair
121{{
122    const char name[kTraceInfoMaxNameLen];
123    TraceInfo info;
124}};
125constexpr TracePair kTraceInfos[kNumTraces] = {{
126{trace_infos}
127}};
128}}
129
130const TraceInfo &GetTraceInfo(const char *traceName)
131{{
132    // Could be improved using std::lower_bound.
133    for (const TracePair &tracePair : kTraceInfos)
134    {{
135        if (strncmp(tracePair.name, traceName, kTraceInfoMaxNameLen) == 0)
136        {{
137            return tracePair.info;
138        }}
139    }}
140    UNREACHABLE();
141    return kTraceInfos[0].info;
142}}
143}}  // namespace trace_angle
144"""
145
146CIPD_TRACE_PREFIX = 'angle/traces'
147EXPERIMENTAL_CIPD_PREFIX = 'experimental/google.com/%s/angle/traces'
148DEPS_PATH = '../../../DEPS'
149DEPS_START = '# === ANGLE Restricted Trace Generated Code Start ==='
150DEPS_END = '# === ANGLE Restricted Trace Generated Code End ==='
151DEPS_TEMPLATE = """\
152  'src/tests/restricted_traces/{trace}': {{
153      'packages': [
154        {{
155            'package': '{trace_prefix}/{trace}',
156            'version': 'version:{version}',
157        }},
158      ],
159      'dep_type': 'cipd',
160      'condition': 'checkout_angle_restricted_traces',
161  }},
162"""
163
164
165def reject_duplicate_keys(pairs):
166    found_keys = {}
167    for key, value in pairs:
168        if key in found_keys:
169            raise ValueError("duplicate key: %r" % (key,))
170        else:
171            found_keys[key] = value
172    return found_keys
173
174
175# TODO(http://anglebug.com/5878): Revert back to non-autogen'ed file names for the angledata.gz.
176def get_angledata_filename(trace):
177    angledata_files = glob.glob('%s/%s*angledata.gz' % (trace, trace))
178    assert len(angledata_files) == 1, "Trace '%s' has %d angledata.gz files" % (
179        trace, len(angledata_files))
180    return angledata_files[0].replace('\\', '/')
181
182
183# TODO(jmadill): Remove the GNI generation. http://anglebug.com/5133
184def gen_gni(traces, gni_file, format_args):
185    test_list = []
186    for trace in traces:
187        context = get_context(trace)
188        angledata_file = get_angledata_filename(trace)
189        txt_file = '%s/%s_capture_context%s_files.txt' % (trace, trace, context)
190        json_file_name = '%s/%s.json' % (trace, trace)
191        if os.path.exists(txt_file):
192            with open(txt_file) as f:
193                files = f.readlines()
194                f.close()
195                source_files = ['"%s/%s"' % (trace, file.strip()) for file in files]
196        else:
197            assert os.path.exists(json_file_name), '%s does not exist' % json_file_name
198            with open(json_file_name) as f:
199                json_data = json.loads(f.read())
200                files = json_data["TraceFiles"]
201
202        source_files = ['"%s/%s"' % (trace, file.strip()) for file in files]
203        data_files = ['"%s"' % angledata_file]
204        if os.path.exists(json_file_name):
205            data_files.append('"%s"' % json_file_name)
206
207        for prefix_expr in ['%s/%s_capture_context%s', '%s/%s_context%s']:
208            prefix = prefix_expr % (trace, trace, context)
209            prefix_cpp = '%s.cpp' % prefix
210            if os.path.exists(prefix_cpp):
211                break
212
213        assert os.path.exists(prefix_cpp), '%s does not exist' % prefix_cpp
214
215        test_list += [
216            '["%s", %s, [%s], [%s], "%s"]' %
217            (trace, context, ','.join(source_files), ','.join(data_files), prefix)
218        ]
219
220    format_args['test_list'] = ',\n'.join(test_list)
221    gni_data = GNI_TEMPLATE.format(**format_args)
222    with open(gni_file, "w") as out_file:
223        out_file.write(gni_data)
224    return True
225
226
227def contains_string(trace, string):
228    """Determines if the trace contains a string"""
229    for file in os.listdir(trace):
230        if fnmatch.fnmatch(file, '*.h'):
231            with open(os.path.join(trace, file)) as f:
232                if string in f.read():
233                    return True
234    return False
235
236
237def contains_context_version(trace):
238    """Determines if the trace contains the major/minor context version"""
239    return contains_string(trace, 'kReplayContextClientMajorVersion')
240
241
242def contains_colorspace(trace):
243    """Determines if the trace contains an EGL surface color space"""
244    return contains_string(trace, 'kReplayDrawSurfaceColorSpace')
245
246
247def json_metadata_exists(trace):
248    return os.path.isfile('%s/%s.json' % (trace, trace))
249
250
251def get_trace_info(trace):
252    # Skip getting trace info if we're using JSON metadata.
253    # TODO: Remove generated code. http://anglebug.com/5133
254    if json_metadata_exists(trace):
255        return ''
256
257    # Some traces don't contain major/minor version, so use defaults
258    info = [f'"{trace}"']
259    if contains_context_version(trace):
260        info += [
261            f'{trace}::kReplayContextClientMajorVersion',
262            f'{trace}::kReplayContextClientMinorVersion'
263        ]
264    else:
265        info += [
266            'kDefaultReplayContextClientMajorVersion', 'kDefaultReplayContextClientMinorVersion'
267        ]
268
269    info += [
270        f'{trace}::kReplayFrameStart', f'{trace}::kReplayFrameEnd',
271        f'{trace}::kReplayDrawSurfaceWidth', f'{trace}::kReplayDrawSurfaceHeight'
272    ]
273
274    if contains_colorspace(trace):
275        info += [f'{trace}::kReplayDrawSurfaceColorSpace']
276    else:
277        info += ['kDefaultReplayDrawSurfaceColorSpace']
278
279    # Add placeholder fields to fix an MSVC warning.
280    info += ['0'] * 8
281    info += ['false'] * 5
282
283    return ", ".join(info)
284
285
286def get_context(trace):
287    """Returns the trace context number."""
288    # TODO(jmadill): Remove the txt scan once migrated. http://anglebug.com/5133
289    # Load up the only header present for each trace
290    for file in os.listdir(trace):
291        if fnmatch.fnmatch(file, '*.txt'):
292            # Strip the extension to isolate the context by scanning
293            # for numbers leading up to the last one, i.e.:
294            #     app_capture_context123_files.txt
295            #                          ^^
296            #                  start---||---end
297            start = len(file) - 11
298            end = start + 1
299            while file[start - 1].isdigit():
300                start -= 1
301            context = file[start:end]
302            assert context.isnumeric(), 'Trace context number is not numeric: %s' % context
303            return context
304
305    expr = re.compile(r'.*_context(\d+).cpp')
306    for file in os.listdir(trace):
307        m = expr.match(file)
308        if m:
309            context = m.group(1)
310            assert context.isnumeric(), 'Trace context number is not numeric: %s' % context
311            return context
312    assert False, 'Failed to find context number for %s' % trace
313
314
315def get_header_name(trace):
316    return '%s/%s_capture_context%s.h' % (trace, trace, get_context(trace))
317
318
319def gen_header(header_file, format_args):
320    header_data = HEADER_TEMPLATE.format(**format_args)
321    with open(header_file, "w") as out_file:
322        out_file.write(header_data)
323    return True
324
325
326def gen_source(source_file, format_args):
327    source_data = SOURCE_TEMPLATE.format(**format_args)
328    with open(source_file, "w") as out_file:
329        out_file.write(source_data)
330    return True
331
332
333def gen_git_ignore(traces):
334    ignores = ['%s/' % trace for trace in traces]
335    with open('.gitignore', 'w') as out_file:
336        out_file.write('\n'.join(sorted(ignores)))
337    return True
338
339
340def read_json(json_file):
341    with open(json_file) as map_file:
342        return json.loads(map_file.read(), object_pairs_hook=reject_duplicate_keys)
343
344
345def update_deps(trace_pairs):
346    # Generate substitution string
347    replacement = ""
348    for (trace, version) in trace_pairs:
349        if 'x' in version:
350            version = version.strip('x')
351            trace_prefix = EXPERIMENTAL_CIPD_PREFIX % getpass.getuser()
352        else:
353            trace_prefix = CIPD_TRACE_PREFIX
354        sub = {'trace': trace, 'version': version, 'trace_prefix': trace_prefix}
355        replacement += DEPS_TEMPLATE.format(**sub)
356
357    # Update DEPS to download CIPD dependencies
358    new_deps = ""
359    with open(DEPS_PATH) as f:
360        in_deps = False
361        for line in f:
362            if in_deps:
363                if DEPS_END in line:
364                    new_deps += replacement
365                    new_deps += line
366                    in_deps = False
367            else:
368                if DEPS_START in line:
369                    new_deps += line
370                    in_deps = True
371                else:
372                    new_deps += line
373        f.close()
374
375    with open(DEPS_PATH, 'w') as f:
376        f.write(new_deps)
377        f.close()
378
379    return True
380
381
382def main():
383    json_file = 'restricted_traces.json'
384    gni_file = 'restricted_traces_autogen.gni'
385    header_file = 'restricted_traces_autogen.h'
386    source_file = 'restricted_traces_autogen.cpp'
387
388    json_data = read_json(json_file)
389    if 'traces' not in json_data:
390        print('Trace data missing traces key.')
391        return 1
392    trace_pairs = [trace.split(' ') for trace in json_data['traces']]
393    traces = [trace_pair[0] for trace_pair in trace_pairs]
394
395    # auto_script parameters.
396    if len(sys.argv) > 1:
397        inputs = [json_file]
398
399        # Note: we do not include DEPS in the list of outputs to simplify the integration.
400        # Otherwise we'd continually need to regenerate on any roll.
401        outputs = [gni_file, header_file, source_file, '.gitignore']
402
403        if sys.argv[1] == 'inputs':
404            print(','.join(inputs))
405        elif sys.argv[1] == 'outputs':
406            print(','.join(outputs))
407        else:
408            print('Invalid script parameters.')
409            return 1
410        return 0
411
412    format_args = {
413        'script_name': os.path.basename(__file__),
414        'data_source_name': json_file,
415    }
416
417    if not gen_gni(traces, gni_file, format_args):
418        print('.gni file generation failed.')
419        return 1
420
421    trace_infos = ['{"%s", {%s}}' % (trace, get_trace_info(trace)) for trace in traces]
422
423    no_json_traces = filter(lambda trace: not json_metadata_exists(trace), traces)
424    includes = ['#include "%s"' % get_header_name(trace) for trace in no_json_traces]
425
426    format_args['filename'] = 'restricted_traces_autogen'
427    format_args['num_traces'] = len(trace_infos)
428    format_args['trace_includes'] = '\n'.join(includes)
429    format_args['trace_infos'] = ',\n'.join(trace_infos)
430    if not gen_header(header_file, format_args):
431        print('.h file generation failed.')
432        return 1
433
434    if not gen_source(source_file, format_args):
435        print('.cpp file generation failed.')
436        return 1
437
438    if not gen_git_ignore(traces):
439        print('.gitignore file generation failed')
440        return 1
441
442    if not update_deps(trace_pairs):
443        print('DEPS file update failed')
444        return 1
445
446    return 0
447
448
449if __name__ == '__main__':
450    sys.exit(main())
451