1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import abc 6import logging 7import os 8import re 9 10import common 11from autotest_lib.client.common_lib import error, utils 12from autotest_lib.client.common_lib.cros import dev_server 13from autotest_lib.client.common_lib.cros import gs_cache_client 14 15 16# Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py. 17 18 19class ControlFileGetter(object): 20 """ 21 Interface for classes that can list and fetch known control files. 22 """ 23 24 __metaclass__ = abc.ABCMeta 25 26 27 @abc.abstractmethod 28 def get_control_file_list(self, suite_name=''): 29 """ 30 Gather a list of paths to control files. 31 32 @param suite_name: The name of a suite we would like control files for. 33 @return A list of file paths. 34 @throws NoControlFileList if there is an error while listing. 35 """ 36 pass 37 38 39 @abc.abstractmethod 40 def get_control_file_contents(self, test_path): 41 """ 42 Given a path to a control file, return its contents. 43 44 @param test_path: the path to the control file. 45 @return the contents of the control file specified by the path. 46 @throws ControlFileNotFound if the file cannot be retrieved. 47 """ 48 pass 49 50 51 @abc.abstractmethod 52 def get_control_file_contents_by_name(self, test_name): 53 """ 54 Given the name of a control file, return its contents. 55 56 @param test_name: the name of the test whose control file is desired. 57 @return the contents of the control file specified by the name. 58 @throws ControlFileNotFound if the file cannot be retrieved. 59 """ 60 pass 61 62 63class SuiteControlFileGetter(ControlFileGetter): 64 """Interface that additionally supports getting by suite.""" 65 66 67 @abc.abstractmethod 68 def get_suite_info(self, suite_name=''): 69 """ 70 Gather the control paths and contents of all the control files. 71 72 @param suite_name: The name of a suite we would like control files for. 73 @return the control paths and contents of all the control files 74 specified by the name. 75 @throws SuiteControlFileException if the info cannot be retrieved. 76 """ 77 pass 78 79 80class CacheingAndFilteringControlFileGetter(ControlFileGetter): 81 """Wraps ControlFileGetter to cache the retrieved control file list and 82 filter out unwanted control files.""" 83 84 CONTROL_FILE_FILTERS = ['src/debian/control'] 85 86 def __init__(self): 87 super(CacheingAndFilteringControlFileGetter, self).__init__() 88 self._files = [] 89 90 91 def get_control_file_list(self, suite_name=''): 92 """ 93 Gather a list of paths to control files. 94 95 Gets a list of control files; populates |self._files| with that list 96 and then returns the paths to all useful and wanted files in the list. 97 98 @param suite_name: The name of a suite we would like control files for. 99 @return A list of file paths. 100 @throws NoControlFileList if there is an error while listing. 101 """ 102 files = self._get_control_file_list(suite_name=suite_name) 103 for cf_filter in self.CONTROL_FILE_FILTERS: 104 files = filter(lambda path: not path.endswith(cf_filter), files) 105 self._files = files 106 return self._files 107 108 109 @abc.abstractmethod 110 def _get_control_file_list(self, suite_name=''): 111 pass 112 113 114 def get_control_file_path(self, test_name): 115 """ 116 Given the name of a control file, return its path. 117 118 Searches through previously-compiled list in |self._files| for a 119 test named |test_name| and returns the contents of the control file 120 for that test if it is found. 121 122 @param test_name: the name of the test whose control file is desired. 123 @return control file path 124 @throws ControlFileNotFound if the file cannot be retrieved. 125 """ 126 if not self._files and not self.get_control_file_list(): 127 raise error.ControlFileNotFound('No control files found.') 128 129 if 'control' not in test_name: 130 regexp = re.compile(os.path.join(test_name, 'control$')) 131 else: 132 regexp = re.compile(test_name + '$') 133 candidates = filter(regexp.search, self._files) 134 if not candidates: 135 raise error.ControlFileNotFound('No control file for ' + test_name) 136 if len(candidates) > 1: 137 raise error.ControlFileNotFound(test_name + ' is not unique.') 138 return candidates[0] 139 140 141 def get_control_file_contents_by_name(self, test_name): 142 """ 143 Given the name of a control file, return its contents. 144 145 Searches through previously-compiled list in |self._files| for a 146 test named |test_name| and returns the contents of the control file 147 for that test if it is found. 148 149 @param test_name: the name of the test whose control file is desired. 150 @return the contents of the control file specified by the name. 151 @throws ControlFileNotFound if the file cannot be retrieved. 152 """ 153 path = self.get_control_file_path(test_name) 154 return self.get_control_file_contents(path) 155 156 157class FileSystemGetter(CacheingAndFilteringControlFileGetter): 158 """ 159 Class that can list and fetch known control files from disk. 160 161 @var _CONTROL_PATTERN: control file name format to match. 162 """ 163 164 _CONTROL_PATTERN = '^control(?:\..+)?$' 165 166 def __init__(self, paths): 167 """ 168 @param paths: base directories to start search. 169 """ 170 super(FileSystemGetter, self).__init__() 171 self._paths = paths 172 173 174 def _is_useful_file(self, name): 175 return '__init__.py' not in name and '.svn' not in name 176 177 178 def _get_control_file_list(self, suite_name=''): 179 """ 180 Gather a list of paths to control files under |self._paths|. 181 182 Searches under |self._paths| for files that match 183 |self._CONTROL_PATTERN|. Populates |self._files| with that list 184 and then returns the paths to all useful files in the list. 185 186 @param suite_name: The name of a suite we would like control files for. 187 @return A list of files that match |self._CONTROL_PATTERN|. 188 @throws NoControlFileList if we find no files. 189 """ 190 if suite_name: 191 logging.debug('Getting control files for a specific suite has ' 192 'not been implemented for FileSystemGetter. ' 193 'Getting all control files instead.') 194 195 196 regexp = re.compile(self._CONTROL_PATTERN) 197 directories = self._paths 198 # Some of our callers are ill-considered and request that we 199 # search all of /usr/local/autotest (crbug.com/771823). 200 # Fixing the callers immediately is somewhere between a 201 # nuisance and hard. So, we have a blacklist, hoping two 202 # wrongs will somehow make it right. 203 blacklist = { 204 'site-packages', 'venv', 'results', 'logs', 'containers', 205 } 206 while len(directories) > 0: 207 directory = directories.pop() 208 if not os.path.exists(directory): 209 continue 210 try: 211 for name in os.listdir(directory): 212 if name in blacklist: 213 continue 214 fullpath = os.path.join(directory, name) 215 if os.path.isfile(fullpath): 216 if regexp.search(name): 217 # if we are a control file 218 self._files.append(fullpath) 219 elif (not os.path.islink(fullpath) 220 and os.path.isdir(fullpath)): 221 directories.append(fullpath) 222 except OSError: 223 # Some directories under results/ like the Chrome Crash 224 # Reports will cause issues when attempted to be searched. 225 logging.error('Unable to search directory %s for control ' 226 'files.', directory) 227 pass 228 if not self._files: 229 msg = 'No control files under ' + ','.join(self._paths) 230 raise error.NoControlFileList(msg) 231 return [f for f in self._files if self._is_useful_file(f)] 232 233 234 def get_control_file_contents(self, test_path): 235 """ 236 Get the contents of the control file at |test_path|. 237 238 @return The contents of the aforementioned file. 239 @throws ControlFileNotFound if the file cannot be retrieved. 240 """ 241 try: 242 return utils.read_file(test_path) 243 except EnvironmentError as (errno, strerror): 244 msg = "Can't retrieve {0}: {1} ({2})".format(test_path, 245 strerror, 246 errno) 247 raise error.ControlFileNotFound(msg) 248 249 250class DevServerGetter(CacheingAndFilteringControlFileGetter, 251 SuiteControlFileGetter): 252 """Class that can list and fetch known control files from DevServer. 253 254 @var _CONTROL_PATTERN: control file name format to match. 255 """ 256 def __init__(self, build, ds): 257 """ 258 @param build: The build from which to get control files. 259 @param ds: An existing dev_server.DevServer object to use. 260 """ 261 super(DevServerGetter, self).__init__() 262 self._dev_server = ds 263 self._build = build 264 265 266 @staticmethod 267 def create(build, ds=None): 268 """Wraps constructor. Can be mocked for testing purposes. 269 @param build: The build from which to get control files. 270 @param ds: An existing dev_server.DevServer object to use 271 (default=None) 272 @returns: New DevServerGetter. 273 """ 274 return DevServerGetter(build, ds) 275 276 277 def _get_control_file_list(self, suite_name=''): 278 """ 279 Gather a list of paths to control files from |self._dev_server|. 280 281 Get a listing of all the control files for |self._build| on 282 |self._dev_server|. Populates |self._files| with that list 283 and then returns paths (under the autotest dir) to them. If suite_name 284 is specified, this method populates |self._files| with the control 285 files from just the specified suite. 286 287 @param suite_name: The name of a suite we would like control files for. 288 @return A list of control file paths. 289 @throws NoControlFileList if there is an error while listing. 290 """ 291 try: 292 return self._dev_server.list_control_files(self._build, 293 suite_name=suite_name) 294 except dev_server.DevServerException as e: 295 raise error.NoControlFileList(e) 296 297 298 def get_control_file_contents(self, test_path): 299 """ 300 Return the contents of |test_path| from |self._dev_server|. 301 302 Get the contents of the control file at |test_path| for |self._build| on 303 |self._dev_server|. 304 305 @return The contents of |test_path|. None on failure. 306 @throws ControlFileNotFound if the file cannot be retrieved. 307 """ 308 try: 309 return self._dev_server.get_control_file(self._build, test_path) 310 except dev_server.DevServerException as e: 311 raise error.ControlFileNotFound(e) 312 313 314 def _list_suite_controls(self, suite_name=''): 315 """ 316 Gather a dict {path:content} of all control files from 317 |self._dev_server|. 318 319 Get a dict of contents of all the control files for |self._build| on 320 |self._dev_server|: path is the key, and the control file content is 321 the value. 322 323 @param suite_name: The name of a suite we would like control files for. 324 @return A dict of paths and contents of all control files. 325 @throws NoControlFileList if there is an error while listing. 326 """ 327 cache_client = gs_cache_client.GsCacheClient(self._dev_server) 328 return cache_client.list_suite_controls(self._build, suite_name) 329 330 331 def get_suite_info(self, suite_name=''): 332 """ 333 Gather info of a list of control files from |self._dev_server|. 334 335 The info is a dict: {control_path: control_file_content} for 336 |self._build| on |self._dev_server|. 337 338 @param suite_name: The name of a suite we would like control files for. 339 @return A dict of paths and contents of all control files: 340 {path1: content1, path2: content2, ..., pathX: contentX} 341 """ 342 file_contents = self._list_suite_controls(suite_name=suite_name) 343 files = file_contents.keys() 344 for cf_filter in self.CONTROL_FILE_FILTERS: 345 files = filter(lambda path: not path.endswith(cf_filter), files) 346 self._files = files 347 return {f: file_contents[f] for f in files} 348