1# Copyright 2020 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Methods related to querying builder information from Buildbucket.""" 5 6from __future__ import print_function 7 8import json 9import logging 10import os 11import subprocess 12from typing import Any, Dict, Iterable, List, Optional, Set, Tuple 13 14import six 15 16from unexpected_passes_common import constants 17from unexpected_passes_common import data_types 18from unexpected_passes_common import multiprocessing_utils 19 20TESTING_BUILDBOT_DIR = os.path.realpath( 21 os.path.join(os.path.dirname(__file__), '..', 'buildbot')) 22INTERNAL_TESTING_BUILDBOT_DIR = os.path.realpath( 23 os.path.join(constants.SRC_INTERNAL_DIR, 'testing', 'buildbot')) 24 25# Public JSON files for internal builders, which should be treated as internal. 26PUBLIC_INTERNAL_JSON_FILES = { 27 'chrome.json', 28 'chrome.gpu.fyi.json', 29 'internal.chrome.fyi.json', 30 'internal.chromeos.fyi.json', 31} 32 33AUTOGENERATED_JSON_KEY = 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT' 34 35FakeBuildersDict = Dict[data_types.BuilderEntry, Set[data_types.BuilderEntry]] 36 37_registered_instance = None 38 39 40def GetInstance() -> 'Builders': 41 return _registered_instance 42 43 44def RegisterInstance(instance: 'Builders') -> None: 45 global _registered_instance 46 assert _registered_instance is None 47 assert isinstance(instance, Builders) 48 _registered_instance = instance 49 50 51def ClearInstance() -> None: 52 global _registered_instance 53 _registered_instance = None 54 55 56class Builders(): 57 def __init__(self, suite: Optional[str], include_internal_builders: bool): 58 """ 59 Args: 60 suite: A string containing particular suite of interest if applicable, 61 such as for Telemetry-based tests. Can be None if not applicable. 62 include_internal_builders: A boolean indicating whether data from 63 internal builders should be used in addition to external ones. 64 """ 65 self._authenticated = False 66 self._suite = suite 67 self._include_internal_builders = include_internal_builders 68 69 def _ProcessJsonFiles(self, files: List[str], are_internal_files: bool, 70 builder_type: str) -> Set[data_types.BuilderEntry]: 71 builders = set() 72 for filepath in files: 73 if not filepath.endswith('.json'): 74 continue 75 if builder_type == constants.BuilderTypes.CI: 76 if 'tryserver' in filepath: 77 continue 78 elif builder_type == constants.BuilderTypes.TRY: 79 if 'tryserver' not in filepath: 80 continue 81 with open(filepath, encoding='utf-8') as f: 82 buildbot_json = json.load(f) 83 # Skip any JSON files that don't contain builder information. 84 if AUTOGENERATED_JSON_KEY not in buildbot_json: 85 continue 86 87 for builder, test_map in buildbot_json.items(): 88 # Remove the auto-generated comments. 89 if 'AAAA' in builder: 90 continue 91 # Filter out any builders that don't run the suite in question. 92 if not self._BuilderRunsTestOfInterest(test_map): 93 continue 94 builders.add( 95 data_types.BuilderEntry(builder, builder_type, are_internal_files)) 96 return builders 97 98 def GetCiBuilders(self) -> Set[data_types.BuilderEntry]: 99 """Gets the set of CI builders to query. 100 101 Returns: 102 A set of data_types.BuilderEntry, each element corresponding to either a 103 public or internal CI builder to query results from. 104 """ 105 ci_builders = set() 106 107 logging.info('Getting CI builders') 108 ci_builders = self._ProcessJsonFiles(_GetPublicJsonFiles(), False, 109 constants.BuilderTypes.CI) 110 if self._include_internal_builders: 111 ci_builders |= self._ProcessJsonFiles(_GetInternalJsonFiles(), True, 112 constants.BuilderTypes.CI) 113 114 logging.debug('Got %d CI builders after trimming: %s', len(ci_builders), 115 ', '.join([b.name for b in ci_builders])) 116 return ci_builders 117 118 def _BuilderRunsTestOfInterest(self, test_map: Dict[str, Any]) -> bool: 119 """Determines if a builder runs a test of interest. 120 121 Args: 122 test_map: A dict, corresponding to a builder's test spec from a 123 //testing/buildbot JSON file. 124 suite: A string containing particular suite of interest if applicable, 125 such as for Telemetry-based tests. Can be None if not applicable. 126 127 Returns: 128 True if |test_map| contains a test of interest, else False. 129 """ 130 raise NotImplementedError() 131 132 def GetTryBuilders(self, ci_builders: Iterable[data_types.BuilderEntry] 133 ) -> Set[data_types.BuilderEntry]: 134 """Gets the set of try builders to query. 135 136 A try builder is of interest if it mirrors a builder in |ci_builders| or is 137 a dedicated try builder. 138 139 Args: 140 ci_builders: An iterable of data_types.BuilderEntry, each element being a 141 public or internal CI builder that results will be/were queried from. 142 143 Returns: 144 A set of data_types.BuilderEntry, each element being the name of a 145 Chromium try builder to query results from. 146 """ 147 logging.info('Getting try builders') 148 dedicated_try_builders = self._ProcessJsonFiles([ 149 os.path.join(TESTING_BUILDBOT_DIR, f) 150 for f in os.listdir(TESTING_BUILDBOT_DIR) 151 ], False, constants.BuilderTypes.TRY) 152 if self._include_internal_builders: 153 dedicated_try_builders |= self._ProcessJsonFiles([ 154 os.path.join(INTERNAL_TESTING_BUILDBOT_DIR, f) 155 for f in os.listdir(INTERNAL_TESTING_BUILDBOT_DIR) 156 ], True, constants.BuilderTypes.TRY) 157 mirrored_builders = set() 158 no_output_builders = set() 159 160 with multiprocessing_utils.GetProcessPoolContext() as pool: 161 results = pool.map(self._GetMirroredBuildersForCiBuilder, ci_builders) 162 for (builders, found_mirror) in results: 163 if found_mirror: 164 mirrored_builders |= builders 165 else: 166 no_output_builders |= builders 167 168 if no_output_builders: 169 raise RuntimeError( 170 'Did not get Buildbucket output for the following builders. They may ' 171 'need to be added to the GetFakeCiBuilders or ' 172 'GetNonChromiumBuilders .\n%s' % 173 '\n'.join([b.name for b in no_output_builders])) 174 logging.debug('Got %d try builders: %s', len(mirrored_builders), 175 mirrored_builders) 176 return dedicated_try_builders | mirrored_builders 177 178 def _GetMirroredBuildersForCiBuilder( 179 self, ci_builder: data_types.BuilderEntry 180 ) -> Tuple[Set[data_types.BuilderEntry], bool]: 181 """Gets the set of try builders that mirror a CI builder. 182 183 Args: 184 ci_builder: A data_types.BuilderEntry for a public or internal CI builder. 185 186 Returns: 187 A tuple (builders, found_mirror). |builders| is a set of 188 data_types.BuilderEntry, either the set of try builders that mirror 189 |ci_builder| or |ci_builder|, depending on the value of |found_mirror|. 190 |found_mirror| is True if mirrors were actually found, in which case 191 |builders| contains the try builders. Otherwise, |found_mirror| is False 192 and |builders| contains |ci_builder|. 193 """ 194 mirrored_builders = set() 195 if ci_builder in self.GetNonChromiumBuilders(): 196 logging.debug('%s is a non-Chromium CI builder', ci_builder.name) 197 return mirrored_builders, True 198 199 fake_builders = self.GetFakeCiBuilders() 200 if ci_builder in fake_builders: 201 mirrored_builders |= fake_builders[ci_builder] 202 logging.debug('%s is a fake CI builder mirrored by %s', ci_builder.name, 203 ', '.join(b.name for b in fake_builders[ci_builder])) 204 return mirrored_builders, True 205 206 bb_output = self._GetBuildbucketOutputForCiBuilder(ci_builder) 207 if not bb_output: 208 mirrored_builders.add(ci_builder) 209 logging.debug('Did not get Buildbucket output for builder %s', 210 ci_builder.name) 211 return mirrored_builders, False 212 213 bb_json = json.loads(bb_output) 214 mirrored = bb_json.get('output', {}).get('properties', 215 {}).get('mirrored_builders', []) 216 # The mirror names from Buildbucket include the group separated by :, e.g. 217 # tryserver.chromium.android:gpu-fyi-try-android-m-nexus-5x-64, so only grab 218 # the builder name. 219 for mirror in mirrored: 220 split = mirror.split(':') 221 assert len(split) == 2 222 logging.debug('Got mirrored builder for %s: %s', ci_builder.name, 223 split[1]) 224 mirrored_builders.add( 225 data_types.BuilderEntry(split[1], constants.BuilderTypes.TRY, 226 ci_builder.is_internal_builder)) 227 return mirrored_builders, True 228 229 def _GetBuildbucketOutputForCiBuilder(self, 230 ci_builder: data_types.BuilderEntry 231 ) -> str: 232 # Ensure the user is logged in to bb. 233 if not self._authenticated: 234 try: 235 with open(os.devnull, 'w', newline='', encoding='utf-8') as devnull: 236 subprocess.check_call(['bb', 'auth-info'], 237 stdout=devnull, 238 stderr=devnull) 239 except subprocess.CalledProcessError as e: 240 six.raise_from( 241 RuntimeError('You are not logged into bb - run `bb auth-login`.'), 242 e) 243 self._authenticated = True 244 # Split out for ease of testing. 245 # Get the Buildbucket ID for the most recent completed build for a builder. 246 p = subprocess.Popen([ 247 'bb', 248 'ls', 249 '-id', 250 '-1', 251 '-status', 252 'ended', 253 '%s/ci/%s' % (ci_builder.project, ci_builder.name), 254 ], 255 stdout=subprocess.PIPE) 256 # Use the ID to get the most recent build. 257 bb_output = subprocess.check_output([ 258 'bb', 259 'get', 260 '-A', 261 '-json', 262 ], 263 stdin=p.stdout, 264 text=True) 265 return bb_output 266 267 def GetIsolateNames(self) -> Set[str]: 268 """Gets the isolate names that are relevant to this implementation. 269 270 Returns: 271 A set of strings, each element being the name of an isolate of interest. 272 """ 273 raise NotImplementedError() 274 275 def GetFakeCiBuilders(self) -> FakeBuildersDict: 276 """Gets a mapping of fake CI builders to their mirrored trybots. 277 278 Returns: 279 A dict of data_types.BuilderEntry -> set(data_types.BuilderEntry). Each 280 key is a CI builder that doesn't actually exist and each value is a set of 281 try builders that mirror the CI builder but do exist. 282 """ 283 raise NotImplementedError() 284 285 def GetNonChromiumBuilders(self) -> Set[data_types.BuilderEntry]: 286 """Gets the builders that are not actual Chromium builders. 287 288 These are listed in the Chromium //testing/buildbot files, but aren't under 289 the Chromium Buildbucket project. These don't use the same recipes as 290 Chromium builders, and thus don't have the list of trybot mirrors. 291 292 Returns: 293 A set of data_types.BuilderEntry, each element being a non-Chromium 294 builder. 295 """ 296 raise NotImplementedError() 297 298 299def _GetPublicJsonFiles() -> List[str]: 300 return [ 301 os.path.join(TESTING_BUILDBOT_DIR, f) 302 for f in os.listdir(TESTING_BUILDBOT_DIR) 303 if f not in PUBLIC_INTERNAL_JSON_FILES 304 ] 305 306 307def _GetInternalJsonFiles() -> List[str]: 308 internal_files = [ 309 os.path.join(INTERNAL_TESTING_BUILDBOT_DIR, f) 310 for f in os.listdir(INTERNAL_TESTING_BUILDBOT_DIR) 311 ] 312 public_internal_files = [ 313 os.path.join(TESTING_BUILDBOT_DIR, f) 314 for f in os.listdir(TESTING_BUILDBOT_DIR) 315 if f in PUBLIC_INTERNAL_JSON_FILES 316 ] 317 return internal_files + public_internal_files 318