• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# pylint: disable-msg=C0111
2# Copyright 2008 Google Inc. Released under the GPL v2
3
4import warnings
5with warnings.catch_warnings():
6    # The 'compiler' module is gone in Python 3.0.  Let's not say
7    # so in every log file.
8    warnings.simplefilter("ignore", DeprecationWarning)
9    import compiler
10import logging
11import textwrap
12import re
13
14from autotest_lib.client.common_lib import enum
15from autotest_lib.client.common_lib import global_config
16from autotest_lib.client.common_lib import priorities
17
18REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type'])
19OBSOLETE_VARS = set(['experimental'])
20
21CONTROL_TYPE = enum.Enum('Server', 'Client', start_value=1)
22CONTROL_TYPE_NAMES =  enum.Enum(*CONTROL_TYPE.names, string_values=True)
23
24_SUITE_ATTRIBUTE_PREFIX = 'suite:'
25
26CONFIG = global_config.global_config
27
28# Default maximum test result size in kB.
29DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value(
30        'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000)
31
32
33class ControlVariableException(Exception):
34    pass
35
36def _validate_control_file_fields(control_file_path, control_file_vars,
37                                  raise_warnings):
38    """Validate the given set of variables from a control file.
39
40    @param control_file_path: string path of the control file these were
41            loaded from.
42    @param control_file_vars: dict of variables set in a control file.
43    @param raise_warnings: True iff we should raise on invalid variables.
44
45    """
46    diff = REQUIRED_VARS - set(control_file_vars)
47    if diff:
48        warning = ('WARNING: Not all required control '
49                   'variables were specified in %s.  Please define '
50                   '%s.') % (control_file_path, ', '.join(diff))
51        if raise_warnings:
52            raise ControlVariableException(warning)
53        print textwrap.wrap(warning, 80)
54
55    obsolete = OBSOLETE_VARS & set(control_file_vars)
56    if obsolete:
57        warning = ('WARNING: Obsolete variables were '
58                   'specified in %s.  Please remove '
59                   '%s.') % (control_file_path, ', '.join(obsolete))
60        if raise_warnings:
61            raise ControlVariableException(warning)
62        print textwrap.wrap(warning, 80)
63
64
65class ControlData(object):
66    # Available TIME settings in control file, the list must be in lower case
67    # and in ascending order, test running faster comes first.
68    TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy']
69    TEST_TIME = enum.Enum(*TEST_TIME_LIST, string_values=False)
70
71    @staticmethod
72    def get_test_time_index(time):
73        """
74        Get the order of estimated test time, based on the TIME setting in
75        Control file. Faster test gets a lower index number.
76        """
77        try:
78            return ControlData.TEST_TIME.get_value(time.lower())
79        except AttributeError:
80            # Raise exception if time value is not a valid TIME setting.
81            error_msg = '%s is not a valid TIME.' % time
82            logging.error(error_msg)
83            raise ControlVariableException(error_msg)
84
85
86    def __init__(self, vars, path, raise_warnings=False):
87        # Defaults
88        self.path = path
89        self.dependencies = set()
90        # TODO(jrbarnette): This should be removed once outside
91        # code that uses can be changed.
92        self.experimental = False
93        self.run_verify = True
94        self.sync_count = 1
95        self.test_parameters = set()
96        self.test_category = ''
97        self.test_class = ''
98        self.job_retries = 0
99        # Default to require server-side package. Unless require_ssp is
100        # explicitly set to False, server-side package will be used for the
101        # job. This can be overridden by global config
102        # AUTOSERV/enable_ssp_container
103        self.require_ssp = None
104        self.attributes = set()
105        self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB
106        self.priority = priorities.Priority.DEFAULT
107        self.fast = False
108
109        _validate_control_file_fields(self.path, vars, raise_warnings)
110
111        for key, val in vars.iteritems():
112            try:
113                self.set_attr(key, val, raise_warnings)
114            except Exception, e:
115                if raise_warnings:
116                    raise
117                print 'WARNING: %s; skipping' % e
118
119        self._patch_up_suites_from_attributes()
120
121
122    @property
123    def suite_tag_parts(self):
124        """Return the part strings of the test's suite tag."""
125        if hasattr(self, 'suite'):
126            return [part.strip() for part in self.suite.split(',')]
127        else:
128            return []
129
130
131    def set_attr(self, attr, val, raise_warnings=False):
132        attr = attr.lower()
133        try:
134            set_fn = getattr(self, 'set_%s' % attr)
135            set_fn(val)
136        except AttributeError:
137            # This must not be a variable we care about
138            pass
139
140
141    def _patch_up_suites_from_attributes(self):
142        """Patch up the set of suites this test is part of.
143
144        Legacy builds will not have an appropriate ATTRIBUTES field set.
145        Take the union of suites specified via ATTRIBUTES and suites specified
146        via SUITE.
147
148        SUITE used to be its own variable, but now suites are taken only from
149        the attributes.
150
151        """
152
153        suite_names = set()
154        # Extract any suites we know ourselves to be in based on the SUITE
155        # line.  This line is deprecated, but control files in old builds will
156        # still have it.
157        if hasattr(self, 'suite'):
158            existing_suites = self.suite.split(',')
159            existing_suites = [name.strip() for name in existing_suites]
160            existing_suites = [name for name in existing_suites if name]
161            suite_names.update(existing_suites)
162
163        # Figure out if our attributes mention any suites.
164        for attribute in self.attributes:
165            if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX):
166                continue
167            suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):]
168            suite_names.add(suite_name)
169
170        # Rebuild the suite field if necessary.
171        if suite_names:
172            self.set_suite(','.join(sorted(list(suite_names))))
173
174
175    def _set_string(self, attr, val):
176        val = str(val)
177        setattr(self, attr, val)
178
179
180    def _set_option(self, attr, val, options):
181        val = str(val)
182        if val.lower() not in [x.lower() for x in options]:
183            raise ValueError("%s must be one of the following "
184                             "options: %s" % (attr,
185                             ', '.join(options)))
186        setattr(self, attr, val)
187
188
189    def _set_bool(self, attr, val):
190        val = str(val).lower()
191        if val == "false":
192            val = False
193        elif val == "true":
194            val = True
195        else:
196            msg = "%s must be either true or false" % attr
197            raise ValueError(msg)
198        setattr(self, attr, val)
199
200
201    def _set_int(self, attr, val, min=None, max=None):
202        val = int(val)
203        if min is not None and min > val:
204            raise ValueError("%s is %d, which is below the "
205                             "minimum of %d" % (attr, val, min))
206        if max is not None and max < val:
207            raise ValueError("%s is %d, which is above the "
208                             "maximum of %d" % (attr, val, max))
209        setattr(self, attr, val)
210
211
212    def _set_set(self, attr, val):
213        val = str(val)
214        items = [x.strip() for x in val.split(',') if x.strip()]
215        setattr(self, attr, set(items))
216
217
218    def set_author(self, val):
219        self._set_string('author', val)
220
221
222    def set_dependencies(self, val):
223        self._set_set('dependencies', val)
224
225
226    def set_doc(self, val):
227        self._set_string('doc', val)
228
229
230    def set_name(self, val):
231        self._set_string('name', val)
232
233
234    def set_run_verify(self, val):
235        self._set_bool('run_verify', val)
236
237
238    def set_sync_count(self, val):
239        self._set_int('sync_count', val, min=1)
240
241
242    def set_suite(self, val):
243        self._set_string('suite', val)
244
245
246    def set_time(self, val):
247        self._set_option('time', val, ControlData.TEST_TIME_LIST)
248
249
250    def set_test_class(self, val):
251        self._set_string('test_class', val.lower())
252
253
254    def set_test_category(self, val):
255        self._set_string('test_category', val.lower())
256
257
258    def set_test_type(self, val):
259        self._set_option('test_type', val, list(CONTROL_TYPE.names))
260
261
262    def set_test_parameters(self, val):
263        self._set_set('test_parameters', val)
264
265
266    def set_job_retries(self, val):
267        self._set_int('job_retries', val)
268
269
270    def set_bug_template(self, val):
271        if type(val) == dict:
272            setattr(self, 'bug_template', val)
273
274
275    def set_require_ssp(self, val):
276        self._set_bool('require_ssp', val)
277
278
279    def set_build(self, val):
280        self._set_string('build', val)
281
282
283    def set_builds(self, val):
284        if type(val) == dict:
285            setattr(self, 'builds', val)
286
287    def set_max_result_size_kb(self, val):
288        self._set_int('max_result_size_KB', val)
289
290    def set_priority(self, val):
291        self._set_int('priority', val)
292
293    def set_fast(self, val):
294        self._set_bool('fast', val)
295
296    def set_update_type(self, val):
297        self._set_string('update_type', val)
298
299    def set_source_release(self, val):
300        self._set_string('source_release', val)
301
302    def set_target_release(self, val):
303        self._set_string('target_release', val)
304
305    def set_target_payload_uri(self, val):
306        self._set_string('target_payload_uri', val)
307
308    def set_source_payload_uri(self, val):
309        self._set_string('source_payload_uri', val)
310
311    def set_source_archive_uri(self, val):
312        self._set_string('source_archive_uri', val)
313
314    def set_attributes(self, val):
315        # Add subsystem:default if subsystem is not specified.
316        self._set_set('attributes', val)
317        if not any(a.startswith('subsystem') for a in self.attributes):
318            self.attributes.add('subsystem:default')
319
320
321def _extract_const(expr):
322    assert(expr.__class__ == compiler.ast.Const)
323    assert(expr.value.__class__ in (str, int, float, unicode))
324    return str(expr.value).strip()
325
326
327def _extract_dict(expr):
328    assert(expr.__class__ == compiler.ast.Dict)
329    assert(expr.items.__class__ == list)
330    cf_dict = {}
331    for key, value in expr.items:
332        try:
333            key = _extract_const(key)
334            val = _extract_expression(value)
335        except (AssertionError, ValueError):
336            pass
337        else:
338            cf_dict[key] = val
339    return cf_dict
340
341
342def _extract_list(expr):
343    assert(expr.__class__ == compiler.ast.List)
344    list_values = []
345    for value in expr.nodes:
346        try:
347            list_values.append(_extract_expression(value))
348        except (AssertionError, ValueError):
349            pass
350    return list_values
351
352
353def _extract_name(expr):
354    assert(expr.__class__ == compiler.ast.Name)
355    assert(expr.name in ('False', 'True', 'None'))
356    return str(expr.name)
357
358
359def _extract_expression(expr):
360    if expr.__class__ == compiler.ast.Const:
361        return _extract_const(expr)
362    if expr.__class__ == compiler.ast.Name:
363        return _extract_name(expr)
364    if expr.__class__ == compiler.ast.Dict:
365        return _extract_dict(expr)
366    if expr.__class__ == compiler.ast.List:
367        return _extract_list(expr)
368    raise ValueError('Unknown rval %s' % expr)
369
370
371def _extract_assignment(n):
372    assert(n.__class__ == compiler.ast.Assign)
373    assert(n.nodes.__class__ == list)
374    assert(len(n.nodes) == 1)
375    assert(n.nodes[0].__class__ == compiler.ast.AssName)
376    assert(n.nodes[0].flags.__class__ == str)
377    assert(n.nodes[0].name.__class__ == str)
378
379    val = _extract_expression(n.expr)
380    key = n.nodes[0].name.lower()
381
382    return (key, val)
383
384
385def parse_control_string(control, raise_warnings=False, path=''):
386    """Parse a control file from a string.
387
388    @param control: string containing the text of a control file.
389    @param raise_warnings: True iff ControlData should raise an error on
390            warnings about control file contents.
391    @param path: string path to the control file.
392
393    """
394    try:
395        mod = compiler.parse(control)
396    except SyntaxError as e:
397        logging.error('Syntax error (%s) while parsing control string:', e)
398        lines = control.split('\n')
399        for n, l in enumerate(lines):
400            logging.error('Line %d: %s', n + 1, l)
401        raise ControlVariableException("Error parsing data because %s" % e)
402    return finish_parse(mod, path, raise_warnings)
403
404
405def parse_control(path, raise_warnings=False):
406    try:
407        mod = compiler.parseFile(path)
408    except SyntaxError, e:
409        raise ControlVariableException("Error parsing %s because %s" %
410                                       (path, e))
411    return finish_parse(mod, path, raise_warnings)
412
413
414def _try_extract_assignment(node, variables):
415    """Try to extract assignment from the given node.
416
417    @param node: An Assign object.
418    @param variables: Dictionary to store the parsed assignments.
419    """
420    try:
421        key, val = _extract_assignment(node)
422        variables[key] = val
423    except (AssertionError, ValueError):
424        pass
425
426
427def finish_parse(mod, path, raise_warnings):
428    assert(mod.__class__ == compiler.ast.Module)
429    assert(mod.node.__class__ == compiler.ast.Stmt)
430    assert(mod.node.nodes.__class__ == list)
431
432    variables = {}
433    injection_variables = {}
434    for n in mod.node.nodes:
435        if (n.__class__ == compiler.ast.Function and
436            re.match('step\d+', n.name)):
437            vars_in_step = {}
438            for sub_node in n.code.nodes:
439                _try_extract_assignment(sub_node, vars_in_step)
440            if vars_in_step:
441                # Empty the vars collection so assignments from multiple steps
442                # won't be mixed.
443                variables.clear()
444                variables.update(vars_in_step)
445        else:
446            _try_extract_assignment(n, injection_variables)
447
448    variables.update(injection_variables)
449    return ControlData(variables, path, raise_warnings)
450