• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# pylint: disable-msg=C0111
2# Copyright 2008 Google Inc. Released under the GPL v2
3
4import warnings
5with warnings.catch_warnings():
6    # The 'compiler' module is gone in Python 3.0.  Let's not say
7    # so in every log file.
8    warnings.simplefilter("ignore", DeprecationWarning)
9    import compiler
10import logging
11import textwrap
12import re
13
14from autotest_lib.client.common_lib import enum
15from autotest_lib.client.common_lib import global_config
16from autotest_lib.client.common_lib import priorities
17
18REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type'])
19OBSOLETE_VARS = set(['experimental'])
20
21CONTROL_TYPE = enum.Enum('Server', 'Client', start_value=1)
22CONTROL_TYPE_NAMES =  enum.Enum(*CONTROL_TYPE.names, string_values=True)
23
24_SUITE_ATTRIBUTE_PREFIX = 'suite:'
25
26CONFIG = global_config.global_config
27
28# Default maximum test result size in kB.
29DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value(
30        'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000)
31
32
33class ControlVariableException(Exception):
34    pass
35
36def _validate_control_file_fields(control_file_path, control_file_vars,
37                                  raise_warnings):
38    """Validate the given set of variables from a control file.
39
40    @param control_file_path: string path of the control file these were
41            loaded from.
42    @param control_file_vars: dict of variables set in a control file.
43    @param raise_warnings: True iff we should raise on invalid variables.
44
45    """
46    diff = REQUIRED_VARS - set(control_file_vars)
47    if diff:
48        warning = ('WARNING: Not all required control '
49                   'variables were specified in %s.  Please define '
50                   '%s.') % (control_file_path, ', '.join(diff))
51        if raise_warnings:
52            raise ControlVariableException(warning)
53        print textwrap.wrap(warning, 80)
54
55    obsolete = OBSOLETE_VARS & set(control_file_vars)
56    if obsolete:
57        warning = ('WARNING: Obsolete variables were '
58                   'specified in %s.  Please remove '
59                   '%s.') % (control_file_path, ', '.join(obsolete))
60        if raise_warnings:
61            raise ControlVariableException(warning)
62        print textwrap.wrap(warning, 80)
63
64
65class ControlData(object):
66    # Available TIME settings in control file, the list must be in lower case
67    # and in ascending order, test running faster comes first.
68    TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy']
69    TEST_TIME = enum.Enum(*TEST_TIME_LIST, string_values=False)
70
71    @staticmethod
72    def get_test_time_index(time):
73        """
74        Get the order of estimated test time, based on the TIME setting in
75        Control file. Faster test gets a lower index number.
76        """
77        try:
78            return ControlData.TEST_TIME.get_value(time.lower())
79        except AttributeError:
80            # Raise exception if time value is not a valid TIME setting.
81            error_msg = '%s is not a valid TIME.' % time
82            logging.error(error_msg)
83            raise ControlVariableException(error_msg)
84
85
86    def __init__(self, vars, path, raise_warnings=False):
87        # Defaults
88        self.path = path
89        self.dependencies = set()
90        # TODO(jrbarnette): This should be removed once outside
91        # code that uses can be changed.
92        self.experimental = False
93        self.run_verify = True
94        self.sync_count = 1
95        self.test_parameters = set()
96        self.test_category = ''
97        self.test_class = ''
98        self.job_retries = 0
99        # Default to require server-side package. Unless require_ssp is
100        # explicitly set to False, server-side package will be used for the
101        # job. This can be overridden by global config
102        # AUTOSERV/enable_ssp_container
103        self.require_ssp = None
104        self.attributes = set()
105        self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB
106        self.priority = priorities.Priority.DEFAULT
107        self.fast = False
108
109        _validate_control_file_fields(self.path, vars, raise_warnings)
110
111        for key, val in vars.iteritems():
112            try:
113                self.set_attr(key, val, raise_warnings)
114            except Exception, e:
115                if raise_warnings:
116                    raise
117                print 'WARNING: %s; skipping' % e
118
119        self._patch_up_suites_from_attributes()
120
121
122    @property
123    def suite_tag_parts(self):
124        """Return the part strings of the test's suite tag."""
125        if hasattr(self, 'suite'):
126            return [part.strip() for part in self.suite.split(',')]
127        else:
128            return []
129
130
131    def set_attr(self, attr, val, raise_warnings=False):
132        attr = attr.lower()
133        try:
134            set_fn = getattr(self, 'set_%s' % attr)
135            set_fn(val)
136        except AttributeError:
137            # This must not be a variable we care about
138            pass
139
140
141    def _patch_up_suites_from_attributes(self):
142        """Patch up the set of suites this test is part of.
143
144        Legacy builds will not have an appropriate ATTRIBUTES field set.
145        Take the union of suites specified via ATTRIBUTES and suites specified
146        via SUITE.
147
148        SUITE used to be its own variable, but now suites are taken only from
149        the attributes.
150
151        """
152
153        suite_names = set()
154        # Extract any suites we know ourselves to be in based on the SUITE
155        # line.  This line is deprecated, but control files in old builds will
156        # still have it.
157        if hasattr(self, 'suite'):
158            existing_suites = self.suite.split(',')
159            existing_suites = [name.strip() for name in existing_suites]
160            existing_suites = [name for name in existing_suites if name]
161            suite_names.update(existing_suites)
162
163        # Figure out if our attributes mention any suites.
164        for attribute in self.attributes:
165            if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX):
166                continue
167            suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):]
168            suite_names.add(suite_name)
169
170        # Rebuild the suite field if necessary.
171        if suite_names:
172            self.set_suite(','.join(sorted(list(suite_names))))
173
174
175    def _set_string(self, attr, val):
176        val = str(val)
177        setattr(self, attr, val)
178
179
180    def _set_option(self, attr, val, options):
181        val = str(val)
182        if val.lower() not in [x.lower() for x in options]:
183            raise ValueError("%s must be one of the following "
184                             "options: %s" % (attr,
185                             ', '.join(options)))
186        setattr(self, attr, val)
187
188
189    def _set_bool(self, attr, val):
190        val = str(val).lower()
191        if val == "false":
192            val = False
193        elif val == "true":
194            val = True
195        else:
196            msg = "%s must be either true or false" % attr
197            raise ValueError(msg)
198        setattr(self, attr, val)
199
200
201    def _set_int(self, attr, val, min=None, max=None):
202        val = int(val)
203        if min is not None and min > val:
204            raise ValueError("%s is %d, which is below the "
205                             "minimum of %d" % (attr, val, min))
206        if max is not None and max < val:
207            raise ValueError("%s is %d, which is above the "
208                             "maximum of %d" % (attr, val, max))
209        setattr(self, attr, val)
210
211
212    def _set_set(self, attr, val):
213        val = str(val)
214        items = [x.strip() for x in val.split(',') if x.strip()]
215        setattr(self, attr, set(items))
216
217
218    def set_author(self, val):
219        self._set_string('author', val)
220
221
222    def set_dependencies(self, val):
223        self._set_set('dependencies', val)
224
225
226    def set_doc(self, val):
227        self._set_string('doc', val)
228
229
230    def set_name(self, val):
231        self._set_string('name', val)
232
233
234    def set_run_verify(self, val):
235        self._set_bool('run_verify', val)
236
237
238    def set_sync_count(self, val):
239        self._set_int('sync_count', val, min=1)
240
241
242    def set_suite(self, val):
243        self._set_string('suite', val)
244
245
246    def set_time(self, val):
247        self._set_option('time', val, ControlData.TEST_TIME_LIST)
248
249
250    def set_test_class(self, val):
251        self._set_string('test_class', val.lower())
252
253
254    def set_test_category(self, val):
255        self._set_string('test_category', val.lower())
256
257
258    def set_test_type(self, val):
259        self._set_option('test_type', val, list(CONTROL_TYPE.names))
260
261
262    def set_test_parameters(self, val):
263        self._set_set('test_parameters', val)
264
265
266    def set_job_retries(self, val):
267        self._set_int('job_retries', val)
268
269
270    def set_bug_template(self, val):
271        if type(val) == dict:
272            setattr(self, 'bug_template', val)
273
274
275    def set_require_ssp(self, val):
276        self._set_bool('require_ssp', val)
277
278
279    def set_build(self, val):
280        self._set_string('build', val)
281
282
283    def set_builds(self, val):
284        if type(val) == dict:
285            setattr(self, 'builds', val)
286
287    def set_max_result_size_kb(self, val):
288        self._set_int('max_result_size_KB', val)
289
290    def set_priority(self, val):
291        self._set_int('priority', val)
292
293    def set_fast(self, val):
294        self._set_bool('fast', val)
295
296    def set_attributes(self, val):
297        # Add subsystem:default if subsystem is not specified.
298        self._set_set('attributes', val)
299        if not any(a.startswith('subsystem') for a in self.attributes):
300            self.attributes.add('subsystem:default')
301
302
303def _extract_const(expr):
304    assert(expr.__class__ == compiler.ast.Const)
305    assert(expr.value.__class__ in (str, int, float, unicode))
306    return str(expr.value).strip()
307
308
309def _extract_dict(expr):
310    assert(expr.__class__ == compiler.ast.Dict)
311    assert(expr.items.__class__ == list)
312    cf_dict = {}
313    for key, value in expr.items:
314        try:
315            key = _extract_const(key)
316            val = _extract_expression(value)
317        except (AssertionError, ValueError):
318            pass
319        else:
320            cf_dict[key] = val
321    return cf_dict
322
323
324def _extract_list(expr):
325    assert(expr.__class__ == compiler.ast.List)
326    list_values = []
327    for value in expr.nodes:
328        try:
329            list_values.append(_extract_expression(value))
330        except (AssertionError, ValueError):
331            pass
332    return list_values
333
334
335def _extract_name(expr):
336    assert(expr.__class__ == compiler.ast.Name)
337    assert(expr.name in ('False', 'True', 'None'))
338    return str(expr.name)
339
340
341def _extract_expression(expr):
342    if expr.__class__ == compiler.ast.Const:
343        return _extract_const(expr)
344    if expr.__class__ == compiler.ast.Name:
345        return _extract_name(expr)
346    if expr.__class__ == compiler.ast.Dict:
347        return _extract_dict(expr)
348    if expr.__class__ == compiler.ast.List:
349        return _extract_list(expr)
350    raise ValueError('Unknown rval %s' % expr)
351
352
353def _extract_assignment(n):
354    assert(n.__class__ == compiler.ast.Assign)
355    assert(n.nodes.__class__ == list)
356    assert(len(n.nodes) == 1)
357    assert(n.nodes[0].__class__ == compiler.ast.AssName)
358    assert(n.nodes[0].flags.__class__ == str)
359    assert(n.nodes[0].name.__class__ == str)
360
361    val = _extract_expression(n.expr)
362    key = n.nodes[0].name.lower()
363
364    return (key, val)
365
366
367def parse_control_string(control, raise_warnings=False, path=''):
368    """Parse a control file from a string.
369
370    @param control: string containing the text of a control file.
371    @param raise_warnings: True iff ControlData should raise an error on
372            warnings about control file contents.
373    @param path: string path to the control file.
374
375    """
376    try:
377        mod = compiler.parse(control)
378    except SyntaxError as e:
379        logging.error('Syntax error (%s) while parsing control string:', e)
380        lines = control.split('\n')
381        for n, l in enumerate(lines):
382            logging.error('Line %d: %s', n + 1, l)
383        raise ControlVariableException("Error parsing data because %s" % e)
384    return finish_parse(mod, path, raise_warnings)
385
386
387def parse_control(path, raise_warnings=False):
388    try:
389        mod = compiler.parseFile(path)
390    except SyntaxError, e:
391        raise ControlVariableException("Error parsing %s because %s" %
392                                       (path, e))
393    return finish_parse(mod, path, raise_warnings)
394
395
396def _try_extract_assignment(node, variables):
397    """Try to extract assignment from the given node.
398
399    @param node: An Assign object.
400    @param variables: Dictionary to store the parsed assignments.
401    """
402    try:
403        key, val = _extract_assignment(node)
404        variables[key] = val
405    except (AssertionError, ValueError):
406        pass
407
408
409def finish_parse(mod, path, raise_warnings):
410    assert(mod.__class__ == compiler.ast.Module)
411    assert(mod.node.__class__ == compiler.ast.Stmt)
412    assert(mod.node.nodes.__class__ == list)
413
414    variables = {}
415    injection_variables = {}
416    for n in mod.node.nodes:
417        if (n.__class__ == compiler.ast.Function and
418            re.match('step\d+', n.name)):
419            vars_in_step = {}
420            for sub_node in n.code.nodes:
421                _try_extract_assignment(sub_node, vars_in_step)
422            if vars_in_step:
423                # Empty the vars collection so assignments from multiple steps
424                # won't be mixed.
425                variables.clear()
426                variables.update(vars_in_step)
427        else:
428            _try_extract_assignment(n, injection_variables)
429
430    variables.update(injection_variables)
431    return ControlData(variables, path, raise_warnings)
432