• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Methods related to querying builder information from Buildbucket."""
5
6from __future__ import print_function
7
8import json
9import logging
10import os
11import subprocess
12from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
13
14import six
15
16from unexpected_passes_common import constants
17from unexpected_passes_common import data_types
18from unexpected_passes_common import multiprocessing_utils
19
20TESTING_BUILDBOT_DIR = os.path.realpath(
21    os.path.join(os.path.dirname(__file__), '..', 'buildbot'))
22INTERNAL_TESTING_BUILDBOT_DIR = os.path.realpath(
23    os.path.join(constants.SRC_INTERNAL_DIR, 'testing', 'buildbot'))
24
25AUTOGENERATED_JSON_KEY = 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT'
26
27FakeBuildersDict = Dict[data_types.BuilderEntry, Set[data_types.BuilderEntry]]
28
29_registered_instance = None
30
31
32def GetInstance() -> 'Builders':
33  return _registered_instance
34
35
36def RegisterInstance(instance: 'Builders') -> None:
37  global _registered_instance
38  assert _registered_instance is None
39  assert isinstance(instance, Builders)
40  _registered_instance = instance
41
42
43def ClearInstance() -> None:
44  global _registered_instance
45  _registered_instance = None
46
47
48class Builders():
49  def __init__(self, suite: Optional[str], include_internal_builders: bool):
50    """
51    Args:
52      suite: A string containing particular suite of interest if applicable,
53          such as for Telemetry-based tests. Can be None if not applicable.
54      include_internal_builders: A boolean indicating whether data from
55          internal builders should be used in addition to external ones.
56    """
57    self._authenticated = False
58    self._suite = suite
59    self._include_internal_builders = include_internal_builders
60
61  def _ProcessJsonFiles(self, files: List[str], are_internal_files: bool,
62                        builder_type: str) -> Set[data_types.BuilderEntry]:
63    builders = set()
64    for filepath in files:
65      if not filepath.endswith('.json'):
66        continue
67      if builder_type == constants.BuilderTypes.CI:
68        if 'tryserver' in filepath:
69          continue
70      elif builder_type == constants.BuilderTypes.TRY:
71        if 'tryserver' not in filepath:
72          continue
73      with open(filepath) as f:
74        buildbot_json = json.load(f)
75      # Skip any JSON files that don't contain builder information.
76      if AUTOGENERATED_JSON_KEY not in buildbot_json:
77        continue
78
79      for builder, test_map in buildbot_json.items():
80        # Remove the auto-generated comments.
81        if 'AAAA' in builder:
82          continue
83        # Filter out any builders that don't run the suite in question.
84        if not self._BuilderRunsTestOfInterest(test_map):
85          continue
86        builders.add(
87            data_types.BuilderEntry(builder, builder_type, are_internal_files))
88    return builders
89
90  def GetCiBuilders(self) -> Set[data_types.BuilderEntry]:
91    """Gets the set of CI builders to query.
92
93    Returns:
94      A set of data_types.BuilderEntry, each element corresponding to either a
95      public or internal CI builder to query results from.
96    """
97    ci_builders = set()
98
99    logging.info('Getting CI builders')
100    ci_builders = self._ProcessJsonFiles([
101        os.path.join(TESTING_BUILDBOT_DIR, f)
102        for f in os.listdir(TESTING_BUILDBOT_DIR)
103    ], False, constants.BuilderTypes.CI)
104    if self._include_internal_builders:
105      ci_builders |= self._ProcessJsonFiles([
106          os.path.join(INTERNAL_TESTING_BUILDBOT_DIR, f)
107          for f in os.listdir(INTERNAL_TESTING_BUILDBOT_DIR)
108      ], True, constants.BuilderTypes.CI)
109
110    logging.debug('Got %d CI builders after trimming: %s', len(ci_builders),
111                  ', '.join([b.name for b in ci_builders]))
112    return ci_builders
113
114  def _BuilderRunsTestOfInterest(self, test_map: Dict[str, Any]) -> bool:
115    """Determines if a builder runs a test of interest.
116
117    Args:
118      test_map: A dict, corresponding to a builder's test spec from a
119          //testing/buildbot JSON file.
120      suite: A string containing particular suite of interest if applicable,
121          such as for Telemetry-based tests. Can be None if not applicable.
122
123    Returns:
124      True if |test_map| contains a test of interest, else False.
125    """
126    raise NotImplementedError()
127
128  def GetTryBuilders(self, ci_builders: Iterable[data_types.BuilderEntry]
129                     ) -> Set[data_types.BuilderEntry]:
130    """Gets the set of try builders to query.
131
132    A try builder is of interest if it mirrors a builder in |ci_builders| or is
133    a dedicated try builder.
134
135    Args:
136      ci_builders: An iterable of data_types.BuilderEntry, each element being a
137          public or internal CI builder that results will be/were queried from.
138
139    Returns:
140      A set of data_types.BuilderEntry, each element being the name of a
141      Chromium try builder to query results from.
142    """
143    logging.info('Getting try builders')
144    dedicated_try_builders = self._ProcessJsonFiles([
145        os.path.join(TESTING_BUILDBOT_DIR, f)
146        for f in os.listdir(TESTING_BUILDBOT_DIR)
147    ], False, constants.BuilderTypes.TRY)
148    if self._include_internal_builders:
149      dedicated_try_builders |= self._ProcessJsonFiles([
150          os.path.join(INTERNAL_TESTING_BUILDBOT_DIR, f)
151          for f in os.listdir(INTERNAL_TESTING_BUILDBOT_DIR)
152      ], True, constants.BuilderTypes.TRY)
153    mirrored_builders = set()
154    no_output_builders = set()
155
156    pool = multiprocessing_utils.GetProcessPool()
157    results = pool.map(self._GetMirroredBuildersForCiBuilder, ci_builders)
158    for (builders, found_mirror) in results:
159      if found_mirror:
160        mirrored_builders |= builders
161      else:
162        no_output_builders |= builders
163
164    if no_output_builders:
165      raise RuntimeError(
166          'Did not get Buildbucket output for the following builders. They may '
167          'need to be added to the GetFakeCiBuilders or '
168          'GetNonChromiumBuilders .\n%s' %
169          '\n'.join([b.name for b in no_output_builders]))
170    logging.debug('Got %d try builders: %s', len(mirrored_builders),
171                  mirrored_builders)
172    return dedicated_try_builders | mirrored_builders
173
174  def _GetMirroredBuildersForCiBuilder(
175      self, ci_builder: data_types.BuilderEntry
176  ) -> Tuple[Set[data_types.BuilderEntry], bool]:
177    """Gets the set of try builders that mirror a CI builder.
178
179    Args:
180      ci_builder: A data_types.BuilderEntry for a public or internal CI builder.
181
182    Returns:
183      A tuple (builders, found_mirror). |builders| is a set of
184      data_types.BuilderEntry, either the set of try builders that mirror
185      |ci_builder| or |ci_builder|, depending on the value of |found_mirror|.
186      |found_mirror| is True if mirrors were actually found, in which case
187      |builders| contains the try builders. Otherwise, |found_mirror| is False
188      and |builders| contains |ci_builder|.
189    """
190    mirrored_builders = set()
191    if ci_builder in self.GetNonChromiumBuilders():
192      logging.debug('%s is a non-Chromium CI builder', ci_builder.name)
193      return mirrored_builders, True
194
195    fake_builders = self.GetFakeCiBuilders()
196    if ci_builder in fake_builders:
197      mirrored_builders |= fake_builders[ci_builder]
198      logging.debug('%s is a fake CI builder mirrored by %s', ci_builder.name,
199                    ', '.join(b.name for b in fake_builders[ci_builder]))
200      return mirrored_builders, True
201
202    bb_output = self._GetBuildbucketOutputForCiBuilder(ci_builder)
203    if not bb_output:
204      mirrored_builders.add(ci_builder)
205      logging.debug('Did not get Buildbucket output for builder %s',
206                    ci_builder.name)
207      return mirrored_builders, False
208
209    bb_json = json.loads(bb_output)
210    mirrored = bb_json.get('output', {}).get('properties',
211                                             {}).get('mirrored_builders', [])
212    # The mirror names from Buildbucket include the group separated by :, e.g.
213    # tryserver.chromium.android:gpu-fyi-try-android-m-nexus-5x-64, so only grab
214    # the builder name.
215    for mirror in mirrored:
216      split = mirror.split(':')
217      assert len(split) == 2
218      logging.debug('Got mirrored builder for %s: %s', ci_builder.name,
219                    split[1])
220      mirrored_builders.add(
221          data_types.BuilderEntry(split[1], constants.BuilderTypes.TRY,
222                                  ci_builder.is_internal_builder))
223    return mirrored_builders, True
224
225  def _GetBuildbucketOutputForCiBuilder(self,
226                                        ci_builder: data_types.BuilderEntry
227                                        ) -> str:
228    # Ensure the user is logged in to bb.
229    if not self._authenticated:
230      try:
231        with open(os.devnull, 'w') as devnull:
232          subprocess.check_call(['bb', 'auth-info'],
233                                stdout=devnull,
234                                stderr=devnull)
235      except subprocess.CalledProcessError as e:
236        six.raise_from(
237            RuntimeError('You are not logged into bb - run `bb auth-login`.'),
238            e)
239      self._authenticated = True
240    # Split out for ease of testing.
241    # Get the Buildbucket ID for the most recent completed build for a builder.
242    p = subprocess.Popen([
243        'bb',
244        'ls',
245        '-id',
246        '-1',
247        '-status',
248        'ended',
249        '%s/ci/%s' % (ci_builder.project, ci_builder.name),
250    ],
251                         stdout=subprocess.PIPE)
252    # Use the ID to get the most recent build.
253    bb_output = subprocess.check_output([
254        'bb',
255        'get',
256        '-A',
257        '-json',
258    ],
259                                        stdin=p.stdout,
260                                        text=True)
261    return bb_output
262
263  def GetIsolateNames(self) -> Set[str]:
264    """Gets the isolate names that are relevant to this implementation.
265
266    Returns:
267      A set of strings, each element being the name of an isolate of interest.
268    """
269    raise NotImplementedError()
270
271  def GetFakeCiBuilders(self) -> FakeBuildersDict:
272    """Gets a mapping of fake CI builders to their mirrored trybots.
273
274    Returns:
275      A dict of data_types.BuilderEntry -> set(data_types.BuilderEntry). Each
276      key is a CI builder that doesn't actually exist and each value is a set of
277      try builders that mirror the CI builder but do exist.
278    """
279    raise NotImplementedError()
280
281  def GetNonChromiumBuilders(self) -> Set[data_types.BuilderEntry]:
282    """Gets the builders that are not actual Chromium builders.
283
284    These are listed in the Chromium //testing/buildbot files, but aren't under
285    the Chromium Buildbucket project. These don't use the same recipes as
286    Chromium builders, and thus don't have the list of trybot mirrors.
287
288    Returns:
289      A set of data_types.BuilderEntry, each element being a non-Chromium
290      builder.
291    """
292    raise NotImplementedError()
293