• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python2
2#
3# Copyright 2019 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# trigger.py:
8#   Helper script for triggering GPU tests on swarming.
9
10import argparse
11import json
12import hashlib
13import logging
14import os
15import re
16import subprocess
17import sys
18
19# This is the same as the trybots.
20DEFAULT_TASK_PRIORITY = 30
21DEFAULT_POOL = 'chromium.tests.gpu'
22DEFAULT_LOG_LEVEL = 'info'
23DEFAULT_REALM = 'chromium:try'
24GOLD_SERVICE_ACCOUNT = 'chrome-gpu-gold@chops-service-accounts.iam.gserviceaccount.com'
25
26
27def parse_args():
28    parser = argparse.ArgumentParser(os.path.basename(sys.argv[0]))
29    parser.add_argument('gn_path', help='path to GN. (e.g. out/Release)')
30    parser.add_argument('test', help='test name. (e.g. angle_end2end_tests)')
31    parser.add_argument('os_dim', help='OS dimension. (e.g. Windows-10)')
32    parser.add_argument('-s', '--shards', default=1, help='number of shards', type=int)
33    parser.add_argument(
34        '-p', '--pool', default=DEFAULT_POOL, help='swarming pool, default is %s.' % DEFAULT_POOL)
35    parser.add_argument('-g', '--gpu', help='GPU dimension. (e.g. intel-hd-630-win10-stable)')
36    parser.add_argument('-t', '--device-type', help='Android device type (e.g. bullhead)')
37    parser.add_argument('-o', '--device-os', help='Android OS.')
38    parser.add_argument(
39        '-l',
40        '--log',
41        default=DEFAULT_LOG_LEVEL,
42        help='Log level. Default is %s.' % DEFAULT_LOG_LEVEL)
43    parser.add_argument(
44        '--gold', action='store_true', help='Use swarming arguments for Gold tests.')
45    parser.add_argument(
46        '--priority',
47        help='Task priority. Default is %s. Use judiciously.' % DEFAULT_TASK_PRIORITY,
48        default=DEFAULT_TASK_PRIORITY)
49    parser.add_argument(
50        '-e',
51        '--env',
52        action='append',
53        default=[],
54        help='Environment variables. Can be specified multiple times.')
55
56    return parser.parse_known_args()
57
58
59def invoke_mb(args):
60    mb_script_path = os.path.join('tools', 'mb', 'mb.py')
61    mb_args = ['python', mb_script_path] + args
62
63    # Attempt to detect standalone vs chromium component build.
64    is_standalone = not os.path.isdir(os.path.join('third_party', 'angle'))
65
66    if is_standalone:
67        logging.info('Standalone mode detected.')
68        mb_args += ['-i', os.path.join('infra', 'specs', 'gn_isolate_map.pyl')]
69
70    logging.info('Invoking mb: %s' % ' '.join(mb_args))
71    return subprocess.check_output(mb_args)
72
73
74def main():
75    args, unknown = parse_args()
76
77    logging.basicConfig(level=args.log.upper())
78
79    path = args.gn_path.replace('\\', '/')
80    out_gn_path = '//' + path
81    out_file_path = os.path.join(*path.split('/'))
82
83    get_command_output = invoke_mb(['get-swarming-command', out_gn_path, args.test, '--as-list'])
84    swarming_cmd = json.loads(get_command_output)
85    logging.info('Swarming command: %s' % ' '.join(swarming_cmd))
86
87    invoke_mb(['isolate', out_gn_path, args.test])
88
89    isolate_cmd_path = os.path.join('tools', 'luci-go', 'isolate')
90    isolate_file = os.path.join(out_file_path, '%s.isolate' % args.test)
91    archive_file = os.path.join(out_file_path, '%s.archive.json' % args.test)
92
93    isolate_args = [
94        isolate_cmd_path, 'archive', '-i', isolate_file, '-cas-instance', 'chromium-swarm',
95        '-dump-json', archive_file
96    ]
97    logging.info('Invoking isolate: %s' % ' '.join(isolate_args))
98    subprocess.check_call(isolate_args)
99    with open(archive_file) as f:
100        digest = json.load(f).get(args.test)
101
102    logging.info('Got an CAS digest %s' % digest)
103    swarming_script_path = os.path.join('tools', 'luci-go', 'swarming')
104
105    swarming_args = [
106        swarming_script_path, 'trigger', '-S', 'chromium-swarm.appspot.com', '-d',
107        'os=' + args.os_dim, '-d', 'pool=' + args.pool, '-digest', digest
108    ]
109
110    # Set priority. Don't abuse this!
111    swarming_args += ['-priority', str(args.priority), '-realm', DEFAULT_REALM]
112
113    # Define a user tag.
114    try:
115        whoami = subprocess.check_output(['whoami'])
116        # Strip extra stuff (e.g. on Windows we are 'hostname\username')
117        whoami = re.sub(r'\w+[^\w]', '', whoami.strip())
118        swarming_args += ['-user', whoami]
119    except:
120        pass
121
122    if args.gpu:
123        swarming_args += ['-d', 'gpu=' + args.gpu]
124
125    if args.device_type:
126        swarming_args += ['-d', 'device_type=' + args.device_type]
127
128    if args.device_os:
129        swarming_args += ['-d', 'device_os=' + args.device_os]
130
131    cmd_args = ['-relative-cwd', args.gn_path, '--']
132
133    if args.gold:
134        swarming_args += ['-service-account', GOLD_SERVICE_ACCOUNT]
135        cmd_args += ['luci-auth', 'context', '--']
136
137    for env in args.env:
138        swarming_args += ['-env', env]
139
140    cmd_args += swarming_cmd
141
142    if unknown:
143        cmd_args += unknown
144
145    if args.shards > 1:
146        for i in range(args.shards):
147            shard_args = swarming_args[:]
148            shard_args.extend([
149                '--env',
150                'GTEST_TOTAL_SHARDS=%d' % args.shards,
151                '--env',
152                'GTEST_SHARD_INDEX=%d' % i,
153            ])
154
155            shard_args += cmd_args
156
157            logging.info('Invoking swarming: %s' % ' '.join(shard_args))
158            subprocess.call(shard_args)
159    else:
160        swarming_args += cmd_args
161        logging.info('Invoking swarming: %s' % ' '.join(swarming_args))
162        subprocess.call(swarming_args)
163    return 0
164
165
166if __name__ == '__main__':
167    sys.exit(main())
168