• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2#
3# Copyright 2021 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# sync_restricted_traces_to_cipd.py:
8#   Ensures the restricted traces are uploaded to CIPD. Versions are encoded in
9#   restricted_traces.json. Requires access to the CIPD path to work.
10
11import argparse
12from concurrent import futures
13import getpass
14import fnmatch
15import logging
16import json
17import os
18import platform
19import signal
20import subprocess
21import sys
22
23CIPD_PREFIX = 'angle/traces'
24EXPERIMENTAL_CIPD_PREFIX = 'experimental/google.com/%s/angle/traces'
25LOG_LEVEL = 'info'
26JSON_PATH = 'restricted_traces.json'
27SCRIPT_DIR = os.path.dirname(sys.argv[0])
28MAX_THREADS = 8
29LONG_TIMEOUT = 100000
30
31
32def cipd(args, suppress_stdout=True):
33    logging.debug('running cipd with args: %s', ' '.join(args))
34    exe = 'cipd.bat' if platform.system() == 'Windows' else 'cipd'
35    if suppress_stdout:
36        # Capture stdout, only log if --log=debug after the process terminates
37        process = subprocess.run([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
38        if process.stdout:
39            logging.debug('cipd stdout:\n%s' % process.stdout.decode())
40    else:
41        # Stdout is piped to the caller's stdout, visible immediately
42        process = subprocess.run([exe] + args)
43    return process.returncode
44
45
46def cipd_name_and_version(trace, trace_version):
47    if 'x' in trace_version:
48        trace_prefix = EXPERIMENTAL_CIPD_PREFIX % getpass.getuser()
49        trace_version = trace_version.strip('x')
50    else:
51        trace_prefix = CIPD_PREFIX
52
53    trace_name = '%s/%s' % (trace_prefix, trace)
54
55    return trace_name, trace_version
56
57
58def check_trace_exists(args, trace, trace_version):
59    cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version)
60
61    # Determine if this version exists
62    return cipd(['describe', cipd_trace_name, '-version', 'version:%s' % cipd_trace_version]) == 0
63
64
65def upload_trace(args, trace, trace_version):
66    trace_folder = os.path.join(SCRIPT_DIR, trace)
67    cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version)
68    cipd_args = ['create', '-name', cipd_trace_name]
69    cipd_args += ['-in', trace_folder]
70    cipd_args += ['-tag', 'version:%s' % cipd_trace_version]
71    cipd_args += ['-log-level', args.log.lower()]
72    cipd_args += ['-install-mode', 'copy']
73    if cipd(cipd_args, suppress_stdout=False) != 0:
74        logging.error('%s version %s: cipd create failed', trace, trace_version)
75        sys.exit(1)
76
77    logging.info('Uploaded trace to cipd: %s version:%s', cipd_trace_name, cipd_trace_version)
78
79
80def check_trace_before_upload(trace):
81    for root, dirs, files in os.walk(os.path.join(SCRIPT_DIR, trace)):
82        if dirs:
83            logging.error('Sub-directories detected for trace %s: %s' % (trace, dirs))
84            sys.exit(1)
85        trace_json = trace + '.json'
86        with open(os.path.join(root, trace_json)) as f:
87            jtrace = json.load(f)
88        additional_files = set([trace_json, trace + '.angledata.gz'])
89        extra_files = set(files) - set(jtrace['TraceFiles']) - additional_files
90        if extra_files:
91            logging.error('Unexpected files, not listed in %s.json [TraceFiles]:\n%s', trace,
92                          '\n'.join(extra_files))
93            sys.exit(1)
94
95
96def main(args):
97    logging.basicConfig(level=args.log.upper())
98
99    with open(os.path.join(SCRIPT_DIR, JSON_PATH)) as f:
100        traces = json.loads(f.read())
101
102    logging.info('Checking cipd for existing versions (this takes time without --filter)')
103    f_exists = {}
104    trace_versions = {}
105    with futures.ThreadPoolExecutor(max_workers=args.threads) as executor:
106        for trace_info in traces['traces']:
107            trace, trace_version = trace_info.split(' ')
108            trace_versions[trace] = trace_version
109            if args.filter and not fnmatch.fnmatch(trace, args.filter):
110                logging.debug('Skipping %s because it does not match the test filter.' % trace)
111                continue
112            assert trace not in f_exists
113            f_exists[trace] = executor.submit(check_trace_exists, args, trace, trace_version)
114
115    to_upload = [trace for trace, f in f_exists.items() if not f.result()]
116    if not to_upload:
117        logging.info('All traces are in sync with cipd')
118        return 0
119
120    logging.info('The following traces are out of sync with cipd:')
121    for trace in to_upload:
122        print(' ', trace, trace_versions[trace])
123        check_trace_before_upload(trace)
124
125    if args.upload or input('Upload [y/N]?') == 'y':
126        for trace in to_upload:
127            upload_trace(args, trace, trace_versions[trace])
128    else:
129        logging.error('Aborted')
130        return 1
131
132    return 0
133
134
135if __name__ == '__main__':
136    parser = argparse.ArgumentParser()
137    parser.add_argument(
138        '-p', '--prefix', help='CIPD Prefix. Default: %s' % CIPD_PREFIX, default=CIPD_PREFIX)
139    parser.add_argument(
140        '-l', '--log', help='Logging level. Default: %s' % LOG_LEVEL, default=LOG_LEVEL)
141    parser.add_argument(
142        '-f', '--filter', help='Only sync specified tests. Supports fnmatch expressions.')
143    parser.add_argument(
144        '-t',
145        '--threads',
146        help='Maxiumum parallel threads. Default: %s' % MAX_THREADS,
147        default=MAX_THREADS)
148    parser.add_argument('--upload', action='store_true', help='Upload without asking.')
149    args = parser.parse_args()
150
151    logging.basicConfig(level=args.log.upper())
152
153    sys.exit(main(args))
154