• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import abc
6import logging
7import os
8import re
9
10import common
11from autotest_lib.client.common_lib import error, utils
12from autotest_lib.client.common_lib.cros import dev_server
13from autotest_lib.client.common_lib.cros import gs_cache_client
14
15
16# Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py.
17
18
19class ControlFileGetter(object):
20    """
21    Interface for classes that can list and fetch known control files.
22    """
23
24    __metaclass__ = abc.ABCMeta
25
26
27    @abc.abstractmethod
28    def get_control_file_list(self, suite_name=''):
29        """
30        Gather a list of paths to control files.
31
32        @param suite_name: The name of a suite we would like control files for.
33        @return A list of file paths.
34        @throws NoControlFileList if there is an error while listing.
35        """
36        pass
37
38
39    @abc.abstractmethod
40    def get_control_file_contents(self, test_path):
41        """
42        Given a path to a control file, return its contents.
43
44        @param test_path: the path to the control file.
45        @return the contents of the control file specified by the path.
46        @throws ControlFileNotFound if the file cannot be retrieved.
47        """
48        pass
49
50
51    @abc.abstractmethod
52    def get_control_file_contents_by_name(self, test_name):
53        """
54        Given the name of a control file, return its contents.
55
56        @param test_name: the name of the test whose control file is desired.
57        @return the contents of the control file specified by the name.
58        @throws ControlFileNotFound if the file cannot be retrieved.
59        """
60        pass
61
62
63class SuiteControlFileGetter(ControlFileGetter):
64    """Interface that additionally supports getting by suite."""
65
66
67    @abc.abstractmethod
68    def get_suite_info(self, suite_name=''):
69        """
70        Gather the control paths and contents of all the control files.
71
72        @param suite_name: The name of a suite we would like control files for.
73        @return the control paths and contents of all the control files
74        specified by the name.
75        @throws SuiteControlFileException if the info cannot be retrieved.
76        """
77        pass
78
79
80class CacheingAndFilteringControlFileGetter(ControlFileGetter):
81    """Wraps ControlFileGetter to cache the retrieved control file list and
82    filter out unwanted control files."""
83
84    CONTROL_FILE_FILTERS = ['src/debian/control']
85
86    def __init__(self):
87        super(CacheingAndFilteringControlFileGetter, self).__init__()
88        self._files = []
89
90
91    def get_control_file_list(self, suite_name=''):
92        """
93        Gather a list of paths to control files.
94
95        Gets a list of control files; populates |self._files| with that list
96        and then returns the paths to all useful and wanted files in the list.
97
98        @param suite_name: The name of a suite we would like control files for.
99        @return A list of file paths.
100        @throws NoControlFileList if there is an error while listing.
101        """
102        files = self._get_control_file_list(suite_name=suite_name)
103        for cf_filter in self.CONTROL_FILE_FILTERS:
104          files = filter(lambda path: not path.endswith(cf_filter), files)
105        self._files = files
106        return self._files
107
108
109    @abc.abstractmethod
110    def _get_control_file_list(self, suite_name=''):
111        pass
112
113
114    def get_control_file_path(self, test_name):
115        """
116        Given the name of a control file, return its path.
117
118        Searches through previously-compiled list in |self._files| for a
119        test named |test_name| and returns the contents of the control file
120        for that test if it is found.
121
122        @param test_name: the name of the test whose control file is desired.
123        @return control file path
124        @throws ControlFileNotFound if the file cannot be retrieved.
125        """
126        if not self._files and not self.get_control_file_list():
127            raise error.ControlFileNotFound('No control files found.')
128
129        if 'control' not in test_name:
130            regexp = re.compile(os.path.join(test_name, 'control$'))
131        else:
132            regexp = re.compile(test_name + '$')
133        candidates = filter(regexp.search, self._files)
134        if not candidates:
135            raise error.ControlFileNotFound('No control file for ' + test_name)
136        if len(candidates) > 1:
137            raise error.ControlFileNotFound(test_name + ' is not unique.')
138        return candidates[0]
139
140
141    def get_control_file_contents_by_name(self, test_name):
142        """
143        Given the name of a control file, return its contents.
144
145        Searches through previously-compiled list in |self._files| for a
146        test named |test_name| and returns the contents of the control file
147        for that test if it is found.
148
149        @param test_name: the name of the test whose control file is desired.
150        @return the contents of the control file specified by the name.
151        @throws ControlFileNotFound if the file cannot be retrieved.
152        """
153        path = self.get_control_file_path(test_name)
154        return self.get_control_file_contents(path)
155
156
157class FileSystemGetter(CacheingAndFilteringControlFileGetter):
158    """
159    Class that can list and fetch known control files from disk.
160
161    @var _CONTROL_PATTERN: control file name format to match.
162    """
163
164    _CONTROL_PATTERN = '^control(?:\..+)?$'
165
166    def __init__(self, paths):
167        """
168        @param paths: base directories to start search.
169        """
170        super(FileSystemGetter, self).__init__()
171        self._paths = paths
172
173
174    def _is_useful_file(self, name):
175        return '__init__.py' not in name and '.svn' not in name
176
177
178    def _get_control_file_list(self, suite_name=''):
179        """
180        Gather a list of paths to control files under |self._paths|.
181
182        Searches under |self._paths| for files that match
183        |self._CONTROL_PATTERN|.  Populates |self._files| with that list
184        and then returns the paths to all useful files in the list.
185
186        @param suite_name: The name of a suite we would like control files for.
187        @return A list of files that match |self._CONTROL_PATTERN|.
188        @throws NoControlFileList if we find no files.
189        """
190        if suite_name:
191            logging.debug('Getting control files for a specific suite has '
192                          'not been implemented for FileSystemGetter. '
193                          'Getting all control files instead.')
194
195
196        regexp = re.compile(self._CONTROL_PATTERN)
197        directories = self._paths
198        # Some of our callers are ill-considered and request that we
199        # search all of /usr/local/autotest (crbug.com/771823).
200        # Fixing the callers immediately is somewhere between a
201        # nuisance and hard.  So, we have a blacklist, hoping two
202        # wrongs will somehow make it right.
203        blacklist = {
204            'site-packages', 'venv', 'results', 'logs', 'containers',
205        }
206        while len(directories) > 0:
207            directory = directories.pop()
208            if not os.path.exists(directory):
209                continue
210            try:
211                for name in os.listdir(directory):
212                    if name in blacklist:
213                        continue
214                    fullpath = os.path.join(directory, name)
215                    if os.path.isfile(fullpath):
216                        if regexp.search(name):
217                            # if we are a control file
218                            self._files.append(fullpath)
219                    elif (not os.path.islink(fullpath)
220                          and os.path.isdir(fullpath)):
221                        directories.append(fullpath)
222            except OSError:
223                # Some directories under results/ like the Chrome Crash
224                # Reports will cause issues when attempted to be searched.
225                logging.error('Unable to search directory %s for control '
226                              'files.', directory)
227                pass
228        if not self._files:
229            msg = 'No control files under ' + ','.join(self._paths)
230            raise error.NoControlFileList(msg)
231        return [f for f in self._files if self._is_useful_file(f)]
232
233
234    def get_control_file_contents(self, test_path):
235        """
236        Get the contents of the control file at |test_path|.
237
238        @return The contents of the aforementioned file.
239        @throws ControlFileNotFound if the file cannot be retrieved.
240        """
241        try:
242            return utils.read_file(test_path)
243        except EnvironmentError as (errno, strerror):
244            msg = "Can't retrieve {0}: {1} ({2})".format(test_path,
245                                                         strerror,
246                                                         errno)
247            raise error.ControlFileNotFound(msg)
248
249
250class DevServerGetter(CacheingAndFilteringControlFileGetter,
251                      SuiteControlFileGetter):
252    """Class that can list and fetch known control files from DevServer.
253
254    @var _CONTROL_PATTERN: control file name format to match.
255    """
256    def __init__(self, build, ds):
257        """
258        @param build: The build from which to get control files.
259        @param ds: An existing dev_server.DevServer object to use.
260        """
261        super(DevServerGetter, self).__init__()
262        self._dev_server = ds
263        self._build = build
264
265
266    @staticmethod
267    def create(build, ds=None):
268        """Wraps constructor.  Can be mocked for testing purposes.
269        @param build: The build from which to get control files.
270        @param ds: An existing dev_server.DevServer object to use
271                  (default=None)
272        @returns: New DevServerGetter.
273        """
274        return DevServerGetter(build, ds)
275
276
277    def _get_control_file_list(self, suite_name=''):
278        """
279        Gather a list of paths to control files from |self._dev_server|.
280
281        Get a listing of all the control files for |self._build| on
282        |self._dev_server|.  Populates |self._files| with that list
283        and then returns paths (under the autotest dir) to them. If suite_name
284        is specified, this method populates |self._files| with the control
285        files from just the specified suite.
286
287        @param suite_name: The name of a suite we would like control files for.
288        @return A list of control file paths.
289        @throws NoControlFileList if there is an error while listing.
290        """
291        try:
292            return self._dev_server.list_control_files(self._build,
293                                                       suite_name=suite_name)
294        except dev_server.DevServerException as e:
295            raise error.NoControlFileList(e)
296
297
298    def get_control_file_contents(self, test_path):
299        """
300        Return the contents of |test_path| from |self._dev_server|.
301
302        Get the contents of the control file at |test_path| for |self._build| on
303        |self._dev_server|.
304
305        @return The contents of |test_path|.  None on failure.
306        @throws ControlFileNotFound if the file cannot be retrieved.
307        """
308        try:
309            return self._dev_server.get_control_file(self._build, test_path)
310        except dev_server.DevServerException as e:
311            raise error.ControlFileNotFound(e)
312
313
314    def _list_suite_controls(self, suite_name=''):
315        """
316        Gather a dict {path:content} of all control files from
317        |self._dev_server|.
318
319        Get a dict of contents of all the control files for |self._build| on
320        |self._dev_server|: path is the key, and the control file content is
321        the value.
322
323        @param suite_name: The name of a suite we would like control files for.
324        @return A dict of paths and contents of all control files.
325        @throws NoControlFileList if there is an error while listing.
326        """
327        cache_client = gs_cache_client.GsCacheClient(self._dev_server)
328        return cache_client.list_suite_controls(self._build, suite_name)
329
330
331    def get_suite_info(self, suite_name=''):
332        """
333        Gather info of a list of control files from |self._dev_server|.
334
335        The info is a dict: {control_path: control_file_content} for
336        |self._build| on |self._dev_server|.
337
338        @param suite_name: The name of a suite we would like control files for.
339        @return A dict of paths and contents of all control files:
340            {path1: content1, path2: content2, ..., pathX: contentX}
341        """
342        file_contents = self._list_suite_controls(suite_name=suite_name)
343        files = file_contents.keys()
344        for cf_filter in self.CONTROL_FILE_FILTERS:
345            files = filter(lambda path: not path.endswith(cf_filter), files)
346        self._files = files
347        return {f: file_contents[f] for f in files}
348