• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Lint as: python2, python3
2# Copyright 2018 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Shared functions by dynamic_suite/suite.py & skylab_suite/cros_suite.py."""
7
8from __future__ import absolute_import
9from __future__ import division
10from __future__ import print_function
11
12import datetime
13import logging
14import multiprocessing
15import re
16import six
17from six.moves import zip
18
19import common
20
21from autotest_lib.client.common_lib import control_data
22from autotest_lib.client.common_lib import error
23from autotest_lib.client.common_lib import global_config
24from autotest_lib.client.common_lib import time_utils
25from autotest_lib.client.common_lib.cros import dev_server
26from autotest_lib.server.cros import provision
27from autotest_lib.server.cros.dynamic_suite import constants
28from autotest_lib.server.cros.dynamic_suite import control_file_getter
29from autotest_lib.server.cros.dynamic_suite import tools
30
31ENABLE_CONTROLS_IN_BATCH = global_config.global_config.get_config_value(
32        'CROS', 'enable_getting_controls_in_batch', type=bool, default=False)
33
34
35def canonicalize_suite_name(suite_name):
36    """Canonicalize the suite's name.
37
38    @param suite_name: the name of the suite.
39    """
40    # Do not change this naming convention without updating
41    # site_utils.parse_job_name.
42    return 'test_suites/control.%s' % suite_name
43
44
45def _formatted_now():
46    """Format the current datetime."""
47    return datetime.datetime.now().strftime(time_utils.TIME_FMT)
48
49
50def make_builds_from_options(options):
51    """Create a dict of builds for creating a suite job.
52
53    The returned dict maps version label prefixes to build names. Together,
54    each key-value pair describes a complete label.
55
56    @param options: SimpleNamespace from argument parsing.
57
58    @return: dict mapping version label prefixes to build names
59    """
60    builds = {}
61    build_prefix = None
62    if options.build:
63        build_prefix = provision.get_version_label_prefix(options.build)
64        builds[build_prefix] = options.build
65
66    if options.cheets_build:
67        builds[provision.CROS_ANDROID_VERSION_PREFIX] = options.cheets_build
68        if build_prefix == provision.CROS_VERSION_PREFIX:
69            builds[build_prefix] += provision.CHEETS_SUFFIX
70
71    if options.firmware_rw_build:
72        builds[provision.FW_RW_VERSION_PREFIX] = options.firmware_rw_build
73
74    if options.firmware_ro_build:
75        builds[provision.FW_RO_VERSION_PREFIX] = options.firmware_ro_build
76
77    return builds
78
79
80def get_test_source_build(builds, **dargs):
81    """Get the build of test code.
82
83    Get the test source build from arguments. If parameter
84    `test_source_build` is set and has a value, return its value. Otherwise
85    returns the ChromeOS build name if it exists. If ChromeOS build is not
86    specified either, raise SuiteArgumentException.
87
88    @param builds: the builds on which we're running this suite. It's a
89                   dictionary of version_prefix:build.
90    @param **dargs: Any other Suite constructor parameters, as described
91                    in Suite.__init__ docstring.
92
93    @return: The build contains the test code.
94    @raise: SuiteArgumentException if both test_source_build and ChromeOS
95            build are not specified.
96
97    """
98    if dargs.get('test_source_build', None):
99        return dargs['test_source_build']
100
101    cros_build = builds.get(provision.CROS_VERSION_PREFIX, None)
102    if cros_build.endswith(provision.CHEETS_SUFFIX):
103        test_source_build = re.sub(
104                provision.CHEETS_SUFFIX + '$', '', cros_build)
105    else:
106        test_source_build = cros_build
107
108    if not test_source_build:
109        raise error.SuiteArgumentException(
110                'test_source_build must be specified if CrOS build is not '
111                'specified.')
112
113    return test_source_build
114
115
116def stage_build_artifacts(build, hostname=None, artifacts=[]):
117    """
118    Ensure components of |build| necessary for installing images are staged.
119
120    @param build image we want to stage.
121    @param hostname hostname of a dut may run test on. This is to help to locate
122        a devserver closer to duts if needed. Default is None.
123    @param artifacts A list of string artifact name to be staged.
124
125    @raises StageControlFileFailure: if the dev server throws 500 while staging
126        suite control files.
127
128    @return: dev_server.ImageServer instance to use with this build.
129    @return: timings dictionary containing staging start/end times.
130    """
131    timings = {}
132    # Ensure components of |build| necessary for installing images are staged
133    # on the dev server. However set synchronous to False to allow other
134    # components to be downloaded in the background.
135    ds = dev_server.resolve(build, hostname=hostname)
136    ds_name = ds.hostname
137    timings[constants.DOWNLOAD_STARTED_TIME] = _formatted_now()
138    try:
139        artifacts_to_stage = ['test_suites', 'control_files']
140        artifacts_to_stage.extend(artifacts if artifacts else [])
141        ds.stage_artifacts(image=build, artifacts=artifacts_to_stage)
142    except dev_server.DevServerException as e:
143        raise error.StageControlFileFailure(
144                "Failed to stage %s on %s: %s" % (build, ds_name, e))
145    timings[constants.PAYLOAD_FINISHED_TIME] = _formatted_now()
146    return ds, timings
147
148
149def get_control_file_by_build(build, ds, suite_name):
150    """Return control file contents for |suite_name|.
151
152    Query the dev server at |ds| for the control file |suite_name|, included
153    in |build| for |board|.
154
155    @param build: unique name by which to refer to the image from now on.
156    @param ds: a dev_server.DevServer instance to fetch control file with.
157    @param suite_name: canonicalized suite name, e.g. test_suites/control.bvt.
158    @raises ControlFileNotFound if a unique suite control file doesn't exist.
159    @raises NoControlFileList if we can't list the control files at all.
160    @raises ControlFileEmpty if the control file exists on the server, but
161                             can't be read.
162
163    @return the contents of the desired control file.
164    """
165    getter = control_file_getter.DevServerGetter.create(build, ds)
166    devserver_name = ds.hostname
167    # Get the control file for the suite.
168    try:
169        control_file_in = getter.get_control_file_contents_by_name(suite_name)
170    except error.CrosDynamicSuiteException as e:
171        raise type(e)('Failed to get control file for %s '
172                      '(devserver: %s) (error: %s)' %
173                      (build, devserver_name, e))
174    if not control_file_in:
175        raise error.ControlFileEmpty(
176            "Fetching %s returned no data. (devserver: %s)" %
177            (suite_name, devserver_name))
178    # Force control files to only contain ascii characters.
179    try:
180        control_file_in.encode('ascii')
181    except UnicodeDecodeError as e:
182        raise error.ControlFileMalformed(str(e))
183
184    return control_file_in
185
186
187def _should_batch_with(cf_getter):
188    """Return whether control files should be fetched in batch.
189
190    This depends on the control file getter and configuration options.
191
192    If cf_getter is a File system ControlFileGetter, the cf_getter will
193    perform a full parse of the root directory associated with the
194    getter. This is the case when it's invoked from suite_preprocessor.
195
196    If cf_getter is a devserver getter, this will look up the suite_name in a
197    suite to control file map generated at build time, and parses the relevant
198    control files alone. This lookup happens on the devserver, so as far
199    as this method is concerned, both cases are equivalent. If
200    enable_controls_in_batch is switched on, this function will call
201    cf_getter.get_suite_info() to get a dict of control files and
202    contents in batch.
203
204    @param cf_getter: a control_file_getter.ControlFileGetter used to list
205           and fetch the content of control files
206    """
207    return (ENABLE_CONTROLS_IN_BATCH
208            and isinstance(cf_getter, control_file_getter.DevServerGetter))
209
210
211def _get_cf_texts_for_suite_batched(cf_getter, suite_name):
212    """Get control file content for given suite with batched getter.
213
214    See get_cf_texts_for_suite for params & returns.
215    """
216    suite_info = cf_getter.get_suite_info(suite_name=suite_name)
217    files = list(suite_info.keys())
218    filtered_files = _filter_cf_paths(files)
219    for path in filtered_files:
220        yield path, suite_info[path]
221
222
223def _get_cf_texts_for_suite_unbatched(cf_getter, suite_name):
224    """Get control file content for given suite with unbatched getter.
225
226    See get_cf_texts_for_suite for params & returns.
227    """
228    files = cf_getter.get_control_file_list(suite_name=suite_name)
229    filtered_files = _filter_cf_paths(files)
230    for path in filtered_files:
231        yield path, cf_getter.get_control_file_contents(path)
232
233
234def _filter_cf_paths(paths):
235    """Remove certain control file paths.
236
237    @param paths: Iterable of paths
238    @returns: generator yielding paths
239    """
240    matcher = re.compile(r'[^/]+/(deps|profilers)/.+')
241    return (path for path in paths if not matcher.match(path))
242
243
244def get_cf_texts_for_suite(cf_getter, suite_name):
245    """Get control file content for given suite.
246
247    @param cf_getter: A control file getter object, e.g.
248        a control_file_getter.DevServerGetter object.
249    @param suite_name: If specified, this method will attempt to restrain
250                       the search space to just this suite's control files.
251    @returns: generator yielding (path, text) tuples
252    """
253    if _should_batch_with(cf_getter):
254        return _get_cf_texts_for_suite_batched(cf_getter, suite_name)
255    else:
256        return _get_cf_texts_for_suite_unbatched(cf_getter, suite_name)
257
258
259def parse_cf_text(path, text):
260    """Parse control file text.
261
262    @param path: path to control file
263    @param text: control file text contents
264
265    @returns: a ControlData object
266
267    @raises ControlVariableException: There is a syntax error in a
268                                      control file.
269    """
270    test = control_data.parse_control_string(
271            text, raise_warnings=True, path=path)
272    test.text = text
273    return test
274
275def parse_cf_text_process(data):
276    """Worker process for parsing control file text
277
278    @param data: Tuple of path, text, forgiving_error, and test_args.
279
280    @returns: Tuple of the path and test ControlData
281
282    @raises ControlVariableException: If forgiving_error is false parsing
283                                      exceptions are raised instead of logged.
284    """
285    path, text, forgiving_error, test_args = data
286
287    if test_args:
288        text = tools.inject_vars(test_args, text)
289
290    try:
291        found_test = parse_cf_text(path, text)
292    except control_data.ControlVariableException as e:
293        if not forgiving_error:
294            msg = "Failed parsing %s\n%s" % (path, e)
295            raise control_data.ControlVariableException(msg)
296        logging.warning("Skipping %s\n%s", path, e)
297    except Exception as e:
298        logging.error("Bad %s\n%s", path, e)
299        import traceback
300        logging.error(traceback.format_exc())
301    else:
302        return (path, found_test)
303
304
305def get_process_limit():
306    """Limit the number of CPUs to use.
307
308    On a server many autotest instances can run in parallel. Avoid that
309    each of them requests all the CPUs at the same time causing a spike.
310    """
311    return min(8, multiprocessing.cpu_count())
312
313
314def parse_cf_text_many(control_file_texts,
315                       forgiving_error=False,
316                       test_args=None):
317    """Parse control file texts.
318
319    @param control_file_texts: iterable of (path, text) pairs
320    @param test_args: The test args to be injected into test control file.
321
322    @returns: a dictionary of ControlData objects
323    """
324    tests = {}
325
326    control_file_texts_all = list(control_file_texts)
327    if control_file_texts_all:
328        # Construct input data for worker processes. Each row contains the
329        # path, text, forgiving_error configuration, and test arguments.
330        paths, texts = list(zip(*control_file_texts_all))
331        worker_data = list(zip(paths, texts, [forgiving_error] * len(paths),
332                           [test_args] * len(paths)))
333        pool = multiprocessing.Pool(processes=get_process_limit())
334        raw_result_list = pool.map(parse_cf_text_process, worker_data)
335        pool.close()
336        pool.join()
337
338        result_list = _current_py_compatible_files(raw_result_list)
339        tests = dict(result_list)
340
341    return tests
342
343
344def _current_py_compatible_files(control_files):
345    """Given a list of control_files, return a list of compatible files.
346
347    Remove blanks/ctrl files with errors (aka not python3 when running
348    python3 compatible) items so the dict conversion doesn't fail.
349
350    @return: List of control files filtered down to those who are compatible
351             with the current running version of python
352    """
353    result_list = []
354    for item in control_files:
355        if item:
356            result_list.append(item)
357        elif six.PY2:
358            # Only raise the error in python 2 environments, for now. See
359            # crbug.com/990593
360            raise error.ControlFileMalformed(
361                "Blank or invalid control file. See log for details.")
362    return result_list
363
364
365def retrieve_control_data_for_test(cf_getter, test_name):
366    """Retrieve a test's control file.
367
368    @param cf_getter: a control_file_getter.ControlFileGetter object to
369                      list and fetch the control files' content.
370    @param test_name: Name of test to retrieve.
371
372    @raises ControlVariableException: There is a syntax error in a
373                                      control file.
374
375    @returns a ControlData object
376    """
377    path = cf_getter.get_control_file_path(test_name)
378    text = cf_getter.get_control_file_contents(path)
379    return parse_cf_text(path, text)
380
381
382def retrieve_for_suite(cf_getter, suite_name='', forgiving_error=False,
383                       test_args=None):
384    """Scan through all tests and find all tests.
385
386    @param suite_name: If specified, retrieve this suite's control file.
387
388    @raises ControlVariableException: If forgiving_parser is False and there
389                                      is a syntax error in a control file.
390
391    @returns a dictionary of ControlData objects that based on given
392             parameters.
393    """
394    control_file_texts = get_cf_texts_for_suite(cf_getter, suite_name)
395    return parse_cf_text_many(control_file_texts,
396                              forgiving_error=forgiving_error,
397                              test_args=test_args)
398
399
400def filter_tests(tests, predicate=lambda t: True):
401    """Filter child tests with predicates.
402
403    @tests: A dict of ControlData objects as tests.
404    @predicate: A test filter. By default it's None.
405
406    @returns a list of ControlData objects as tests.
407    """
408    logging.info('Parsed %s child test control files.', len(tests))
409    tests = [test for test in six.itervalues(tests) if predicate(test)]
410    tests.sort(key=lambda t:
411               control_data.ControlData.get_test_time_index(t.time),
412               reverse=True)
413    return tests
414
415
416def name_in_tag_predicate(name):
417    """Returns predicate that takes a control file and looks for |name|.
418
419    Builds a predicate that takes in a parsed control file (a ControlData)
420    and returns True if the SUITE tag is present and contains |name|.
421
422    @param name: the suite name to base the predicate on.
423    @return a callable that takes a ControlData and looks for |name| in that
424            ControlData object's suite member.
425    """
426    return lambda t: name in t.suite_tag_parts
427
428
429def test_name_in_list_predicate(name_list):
430    """Returns a predicate that matches control files by test name.
431
432    The returned predicate returns True for control files whose test name
433    is present in name_list.
434    """
435    name_set = set(name_list)
436    return lambda t: t.name in name_set
437