• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import logging
7import re
8import time
9
10from google.appengine.api import urlfetch
11import webapp2
12
13from base import bigquery
14from base import constants
15from common import buildbot
16
17
18class Builds(webapp2.RequestHandler):
19
20  def get(self):
21    urlfetch.set_default_fetch_deadline(300)
22
23    bq = bigquery.BigQuery()
24
25    current_events = []
26    events = []
27    for master_name in constants.MASTER_NAMES:
28      builders = buildbot.Builders(master_name)
29      available_builds = _AvailableBuilds(builders)
30      recorded_builds = _RecordedBuilds(bq, builders, available_builds)
31      for builder in builders:
32        # Filter out recorded builds from available builds.
33        build_numbers = (available_builds[builder.name] -
34                         recorded_builds[builder.name])
35        builder_current_events, builder_events = _TraceEventsForBuilder(
36            builder, build_numbers)
37        current_events += builder_current_events
38        events += builder_events
39
40    jobs = []
41    if current_events:
42      jobs += bq.InsertRowsAsync(
43          constants.DATASET, constants.CURRENT_BUILDS_TABLE,
44          current_events, truncate=True)
45    if events:
46      jobs += bq.InsertRowsAsync(constants.DATASET, constants.BUILDS_TABLE,
47                                 events)
48
49    for job in jobs:
50      bq.PollJob(job, 60 * 20)  # 20 minutes.
51
52
53def _AvailableBuilds(builders):
54  available_builds = {}
55  for builder in builders:
56    if not builder.cached_builds:
57      available_builds[builder.name] = frozenset()
58      continue
59
60    max_build = max(builder.cached_builds)
61    # Buildbot on tryserver.chromium.perf is occasionally including build 0 in
62    # its list of cached builds. That results in more builds than we want.
63    # Limit the list to the last 100 builds, because the urlfetch URL limit is
64    # 2048 bytes, and "&select=100000" * 100 is 1400 bytes.
65    builds = frozenset(build for build in builder.cached_builds
66                       if build >= max_build - 100)
67    available_builds[builder.name] = builds
68  return available_builds
69
70
71def _RecordedBuilds(bq, builders, available_builds):
72  # 105 days / 15 weeks. Must be some number greater than 100 days, because
73  # we request up to 100 builds (see above comment), and the slowest cron bots
74  # run one job every day.
75  start_time_ms = -1000 * 60 * 60 * 24 * 105
76  table = '%s.%s@%d-' % (constants.DATASET, constants.BUILDS_TABLE,
77                         start_time_ms)
78
79  conditions = []
80  for builder in builders:
81    if not available_builds[builder.name]:
82      continue
83    max_build = max(available_builds[builder.name])
84    min_build = min(available_builds[builder.name])
85    conditions.append('WHEN builder = "%s" THEN build >= %d AND build <= %d' %
86                      (builder.name, min_build, max_build))
87
88  query = (
89      'SELECT builder, build '
90      'FROM [%s] ' % table +
91      'WHERE CASE %s END ' % ' '.join(conditions) +
92      'GROUP BY builder, build'
93  )
94  query_result = bq.QuerySync(query, 600)
95
96  builds = collections.defaultdict(set)
97  for row in query_result:
98    builds[row['f'][0]['v']].add(int(row['f'][1]['v']))
99  return builds
100
101
102def _TraceEventsForBuilder(builder, build_numbers):
103  if not build_numbers:
104    return (), ()
105
106  build_numbers_string = ', '.join(map(str, sorted(build_numbers)))
107  logging.info('Getting %s: %s', builder.name, build_numbers_string)
108
109  # Fetch build information and generate trace events.
110  current_events = []
111  events = []
112
113  builder_builds = builder.builds.Fetch(build_numbers)
114  query_time = time.time()
115  for build in builder_builds:
116    if build.complete:
117      events += _TraceEventsFromBuild(builder, build, query_time)
118    else:
119      current_events += _TraceEventsFromBuild(builder, build, query_time)
120
121  return current_events, events
122
123
124def _TraceEventsFromBuild(builder, build, query_time):
125  match = re.match(r'(.+) \(([0-9]+)\)', builder.name)
126  if match:
127    configuration, host_shard = match.groups()
128    host_shard = int(host_shard)
129  else:
130    configuration = builder.name
131    host_shard = 0
132
133  # Build trace event.
134  if build.end_time:
135    build_end_time = build.end_time
136  else:
137    build_end_time = query_time
138  os, os_version, role = _ParseBuilderName(builder.master_name, builder.name)
139  yield {
140      'name': 'Build %d' % build.number,
141      'start_time': build.start_time,
142      'end_time': build_end_time,
143
144      'build': build.number,
145      'builder': builder.name,
146      'configuration': configuration,
147      'host_shard': host_shard,
148      'hostname': build.slave_name,
149      'master': builder.master_name,
150      'os': os,
151      'os_version': os_version,
152      'role': role,
153      'status': build.status,
154      'url': build.url,
155  }
156
157  # Step trace events.
158  for step in build.steps:
159    if not step.start_time:
160      continue
161
162    if step.name == 'steps':
163      continue
164
165    if step.end_time:
166      step_end_time = step.end_time
167    else:
168      step_end_time = query_time
169    yield {
170        'name': step.name,
171        'start_time': step.start_time,
172        'end_time': step_end_time,
173
174        'benchmark': step.name,  # TODO(dtu): This isn't always right.
175        'build': build.number,
176        'builder': builder.name,
177        'configuration': configuration,
178        'host_shard': host_shard,
179        'hostname': build.slave_name,
180        'master': builder.master_name,
181        'os': os,
182        'os_version': os_version,
183        'role': role,
184        'status': step.status,
185        'url': step.url,
186    }
187
188
189def _ParseBuilderName(master_name, builder_name):
190  if master_name == 'chromium.perf':
191    match = re.match(r'^([A-Za-z]+)(?: ([0-9\.]+|XP))?([A-Za-z0-9-\. ]+)? '
192                     r'(Builder|Perf)(?: \([0-9]+\))?$', builder_name).groups()
193    os = match[0]
194    if match[1]:
195      os_version = match[1]
196    else:
197      os_version = None
198    if match[3] == 'Builder':
199      role = 'builder'
200    elif match[3] == 'Perf':
201      role = 'tester'
202    else:
203      raise NotImplementedError()
204  elif master_name == 'client.catapult':
205    match = re.match(r'^Catapult(?: ([A-Za-z])+)? ([A-Za-z]+)$',
206                     builder_name).groups()
207    os = match[1]
208    os_version = None
209    role = match[0]
210    if not role:
211      role = 'tester'
212  elif master_name == 'tryserver.chromium.perf':
213    match = re.match(r'^(android|linux|mac|win).*_([a-z]+)$',
214                     builder_name).groups()
215    os = match[0]
216    os_version = None
217    role = match[1]
218  elif master_name == 'tryserver.client.catapult':
219    match = re.match(r'^Catapult(?: (Android|Linux|Mac|Windows))? ([A-Za-z]+)$',
220                     builder_name).groups()
221    os = match[0]
222    os_version = None
223    role = match[1]
224  else:
225    raise NotImplementedError()
226
227  if os:
228    os = os.lower()
229  if os == 'windows':
230    os = 'win'
231  if os_version:
232    os_version = os_version.lower()
233  role = role.lower()
234
235  return (os, os_version, role)
236