1#!/usr/bin/python3 2# 3# Copyright 2021 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7# sync_restricted_traces_to_cipd.py: 8# Ensures the restricted traces are uploaded to CIPD. Versions are encoded in 9# restricted_traces.json. Requires access to the CIPD path to work. 10 11import argparse 12import getpass 13import fnmatch 14import logging 15import itertools 16import json 17import multiprocessing 18import os 19import platform 20import signal 21import subprocess 22import sys 23 24CIPD_PREFIX = 'angle/traces' 25EXPERIMENTAL_CIPD_PREFIX = 'experimental/google.com/%s/angle/traces' 26LOG_LEVEL = 'info' 27JSON_PATH = 'restricted_traces.json' 28SCRIPT_DIR = os.path.dirname(sys.argv[0]) 29MAX_THREADS = 8 30LONG_TIMEOUT = 100000 31 32EXIT_SUCCESS = 0 33EXIT_FAILURE = 1 34 35 36def cipd(logger, *args): 37 logger.debug('running cipd with args: %s', ' '.join(args)) 38 exe = 'cipd.bat' if platform.system() == 'Windows' else 'cipd' 39 try: 40 completed = subprocess.run([exe] + list(args), stderr=subprocess.STDOUT) 41 except KeyboardInterrupt: 42 pass 43 if completed.stdout: 44 logger.debug('cipd stdout:\n%s' % completed.stdout) 45 return completed.returncode 46 47 48def sync_trace(param): 49 args, trace_info = param 50 logger = args.logger 51 trace, trace_version = trace_info.split(' ') 52 53 if args.filter and not fnmatch.fnmatch(trace, args.filter): 54 logger.debug('Skipping %s because it does not match the test filter.' % trace) 55 return EXIT_SUCCESS 56 57 if 'x' in trace_version: 58 trace_prefix = EXPERIMENTAL_CIPD_PREFIX % getpass.getuser() 59 trace_version = trace_version.strip('x') 60 else: 61 trace_prefix = CIPD_PREFIX 62 63 trace_name = '%s/%s' % (trace_prefix, trace) 64 # Determine if this version exists 65 if cipd(logger, 'describe', trace_name, '-version', 'version:%s' % trace_version) == 0: 66 logger.info('%s version %s already present' % (trace, trace_version)) 67 return EXIT_SUCCESS 68 69 logger.info('%s version %s missing. calling create.' % (trace, trace_version)) 70 trace_folder = os.path.join(SCRIPT_DIR, trace) 71 if cipd(logger, 'create', '-name', trace_name, '-in', trace_folder, '-tag', 'version:%s' % 72 trace_version, '-log-level', args.log.lower(), '-install-mode', 'copy') != 0: 73 logger.error('%s version %s create failed' % (trace, trace_version)) 74 return EXIT_FAILURE 75 return EXIT_SUCCESS 76 77 78def main(args): 79 args.logger = multiprocessing.log_to_stderr() 80 args.logger.setLevel(level=args.log.upper()) 81 82 with open(os.path.join(SCRIPT_DIR, JSON_PATH)) as f: 83 traces = json.loads(f.read()) 84 85 zipped_args = zip(itertools.repeat(args), traces['traces']) 86 87 if args.threads > 1: 88 pool = multiprocessing.Pool(args.threads) 89 try: 90 retval = pool.map_async(sync_trace, zipped_args).get(LONG_TIMEOUT) 91 except KeyboardInterrupt: 92 pool.terminate() 93 pool.join() 94 except Exception as e: 95 print('got exception: %r, terminating the pool' % (e,)) 96 pool.terminate() 97 pool.join() 98 else: 99 retval = map(sync_trace, zipped_args) 100 101 return EXIT_FAILURE if EXIT_FAILURE in retval else EXIT_SUCCESS 102 103 104if __name__ == '__main__': 105 max_threads = min(multiprocessing.cpu_count(), MAX_THREADS) 106 parser = argparse.ArgumentParser() 107 parser.add_argument( 108 '-p', '--prefix', help='CIPD Prefix. Default: %s' % CIPD_PREFIX, default=CIPD_PREFIX) 109 parser.add_argument( 110 '-l', '--log', help='Logging level. Default: %s' % LOG_LEVEL, default=LOG_LEVEL) 111 parser.add_argument( 112 '-f', '--filter', help='Only sync specified tests. Supports fnmatch expressions.') 113 parser.add_argument( 114 '-t', 115 '--threads', 116 help='Maxiumum parallel threads. Default: %s' % max_threads, 117 default=max_threads) 118 args, extra_flags = parser.parse_known_args() 119 120 logging.basicConfig(level=args.log.upper()) 121 122 sys.exit(main(args)) 123