• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# pylint: disable-msg=C0111
2# Copyright 2008 Google Inc. Released under the GPL v2
3
4import warnings
5with warnings.catch_warnings():
6    # The 'compiler' module is gone in Python 3.0.  Let's not say
7    # so in every log file.
8    warnings.simplefilter("ignore", DeprecationWarning)
9    import compiler
10import logging
11import textwrap
12import re
13
14from autotest_lib.client.common_lib import enum
15from autotest_lib.client.common_lib import global_config
16from autotest_lib.client.common_lib import priorities
17
18REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type'])
19OBSOLETE_VARS = set(['experimental'])
20
21CONTROL_TYPE = enum.Enum('Server', 'Client', start_value=1)
22CONTROL_TYPE_NAMES =  enum.Enum(*CONTROL_TYPE.names, string_values=True)
23
24_SUITE_ATTRIBUTE_PREFIX = 'suite:'
25
26CONFIG = global_config.global_config
27
28# Default maximum test result size in kB.
29DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value(
30        'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000)
31
32
33class ControlVariableException(Exception):
34    pass
35
36def _validate_control_file_fields(control_file_path, control_file_vars,
37                                  raise_warnings):
38    """Validate the given set of variables from a control file.
39
40    @param control_file_path: string path of the control file these were
41            loaded from.
42    @param control_file_vars: dict of variables set in a control file.
43    @param raise_warnings: True iff we should raise on invalid variables.
44
45    """
46    diff = REQUIRED_VARS - set(control_file_vars)
47    if diff:
48        warning = ('WARNING: Not all required control '
49                   'variables were specified in %s.  Please define '
50                   '%s.') % (control_file_path, ', '.join(diff))
51        if raise_warnings:
52            raise ControlVariableException(warning)
53        print textwrap.wrap(warning, 80)
54
55    obsolete = OBSOLETE_VARS & set(control_file_vars)
56    if obsolete:
57        warning = ('WARNING: Obsolete variables were '
58                   'specified in %s.  Please remove '
59                   '%s.') % (control_file_path, ', '.join(obsolete))
60        if raise_warnings:
61            raise ControlVariableException(warning)
62        print textwrap.wrap(warning, 80)
63
64
65class ControlData(object):
66    # Available TIME settings in control file, the list must be in lower case
67    # and in ascending order, test running faster comes first.
68    TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy']
69    TEST_TIME = enum.Enum(*TEST_TIME_LIST, string_values=False)
70
71    @staticmethod
72    def get_test_time_index(time):
73        """
74        Get the order of estimated test time, based on the TIME setting in
75        Control file. Faster test gets a lower index number.
76        """
77        try:
78            return ControlData.TEST_TIME.get_value(time.lower())
79        except AttributeError:
80            # Raise exception if time value is not a valid TIME setting.
81            error_msg = '%s is not a valid TIME.' % time
82            logging.error(error_msg)
83            raise ControlVariableException(error_msg)
84
85
86    def __init__(self, vars, path, raise_warnings=False):
87        # Defaults
88        self.path = path
89        self.dependencies = set()
90        # TODO(jrbarnette): This should be removed once outside
91        # code that uses can be changed.
92        self.experimental = False
93        self.run_verify = True
94        self.sync_count = 1
95        self.test_parameters = set()
96        self.test_category = ''
97        self.test_class = ''
98        self.retries = 0
99        self.job_retries = 0
100        # Default to require server-side package. Unless require_ssp is
101        # explicitly set to False, server-side package will be used for the
102        # job. This can be overridden by global config
103        # AUTOSERV/enable_ssp_container
104        self.require_ssp = None
105        self.attributes = set()
106        self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB
107        self.priority = priorities.Priority.DEFAULT
108        self.fast = False
109
110        _validate_control_file_fields(self.path, vars, raise_warnings)
111
112        for key, val in vars.iteritems():
113            try:
114                self.set_attr(key, val, raise_warnings)
115            except Exception, e:
116                if raise_warnings:
117                    raise
118                print 'WARNING: %s; skipping' % e
119
120        self._patch_up_suites_from_attributes()
121
122
123    @property
124    def suite_tag_parts(self):
125        """Return the part strings of the test's suite tag."""
126        if hasattr(self, 'suite'):
127            return [part.strip() for part in self.suite.split(',')]
128        else:
129            return []
130
131
132    def set_attr(self, attr, val, raise_warnings=False):
133        attr = attr.lower()
134        try:
135            set_fn = getattr(self, 'set_%s' % attr)
136            set_fn(val)
137        except AttributeError:
138            # This must not be a variable we care about
139            pass
140
141
142    def _patch_up_suites_from_attributes(self):
143        """Patch up the set of suites this test is part of.
144
145        Legacy builds will not have an appropriate ATTRIBUTES field set.
146        Take the union of suites specified via ATTRIBUTES and suites specified
147        via SUITE.
148
149        SUITE used to be its own variable, but now suites are taken only from
150        the attributes.
151
152        """
153
154        suite_names = set()
155        # Extract any suites we know ourselves to be in based on the SUITE
156        # line.  This line is deprecated, but control files in old builds will
157        # still have it.
158        if hasattr(self, 'suite'):
159            existing_suites = self.suite.split(',')
160            existing_suites = [name.strip() for name in existing_suites]
161            existing_suites = [name for name in existing_suites if name]
162            suite_names.update(existing_suites)
163
164        # Figure out if our attributes mention any suites.
165        for attribute in self.attributes:
166            if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX):
167                continue
168            suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):]
169            suite_names.add(suite_name)
170
171        # Rebuild the suite field if necessary.
172        if suite_names:
173            self.set_suite(','.join(sorted(list(suite_names))))
174
175
176    def _set_string(self, attr, val):
177        val = str(val)
178        setattr(self, attr, val)
179
180
181    def _set_option(self, attr, val, options):
182        val = str(val)
183        if val.lower() not in [x.lower() for x in options]:
184            raise ValueError("%s must be one of the following "
185                             "options: %s" % (attr,
186                             ', '.join(options)))
187        setattr(self, attr, val)
188
189
190    def _set_bool(self, attr, val):
191        val = str(val).lower()
192        if val == "false":
193            val = False
194        elif val == "true":
195            val = True
196        else:
197            msg = "%s must be either true or false" % attr
198            raise ValueError(msg)
199        setattr(self, attr, val)
200
201
202    def _set_int(self, attr, val, min=None, max=None):
203        val = int(val)
204        if min is not None and min > val:
205            raise ValueError("%s is %d, which is below the "
206                             "minimum of %d" % (attr, val, min))
207        if max is not None and max < val:
208            raise ValueError("%s is %d, which is above the "
209                             "maximum of %d" % (attr, val, max))
210        setattr(self, attr, val)
211
212
213    def _set_set(self, attr, val):
214        val = str(val)
215        items = [x.strip() for x in val.split(',') if x.strip()]
216        setattr(self, attr, set(items))
217
218
219    def set_author(self, val):
220        self._set_string('author', val)
221
222
223    def set_dependencies(self, val):
224        self._set_set('dependencies', val)
225
226
227    def set_doc(self, val):
228        self._set_string('doc', val)
229
230
231    def set_name(self, val):
232        self._set_string('name', val)
233
234
235    def set_run_verify(self, val):
236        self._set_bool('run_verify', val)
237
238
239    def set_sync_count(self, val):
240        self._set_int('sync_count', val, min=1)
241
242
243    def set_suite(self, val):
244        self._set_string('suite', val)
245
246
247    def set_time(self, val):
248        self._set_option('time', val, ControlData.TEST_TIME_LIST)
249
250
251    def set_test_class(self, val):
252        self._set_string('test_class', val.lower())
253
254
255    def set_test_category(self, val):
256        self._set_string('test_category', val.lower())
257
258
259    def set_test_type(self, val):
260        self._set_option('test_type', val, list(CONTROL_TYPE.names))
261
262
263    def set_test_parameters(self, val):
264        self._set_set('test_parameters', val)
265
266
267    def set_retries(self, val):
268        self._set_int('retries', val)
269
270
271    def set_job_retries(self, val):
272        self._set_int('job_retries', val)
273
274
275    def set_bug_template(self, val):
276        if type(val) == dict:
277            setattr(self, 'bug_template', val)
278
279
280    def set_require_ssp(self, val):
281        self._set_bool('require_ssp', val)
282
283
284    def set_build(self, val):
285        self._set_string('build', val)
286
287
288    def set_builds(self, val):
289        if type(val) == dict:
290            setattr(self, 'builds', val)
291
292    def set_max_result_size_kb(self, val):
293        self._set_int('max_result_size_KB', val)
294
295    def set_priority(self, val):
296        self._set_int('priority', val)
297
298    def set_fast(self, val):
299        self._set_bool('fast', val)
300
301    def set_attributes(self, val):
302        # Add subsystem:default if subsystem is not specified.
303        self._set_set('attributes', val)
304        if not any(a.startswith('subsystem') for a in self.attributes):
305            self.attributes.add('subsystem:default')
306
307
308def _extract_const(expr):
309    assert(expr.__class__ == compiler.ast.Const)
310    assert(expr.value.__class__ in (str, int, float, unicode))
311    return str(expr.value).strip()
312
313
314def _extract_dict(expr):
315    assert(expr.__class__ == compiler.ast.Dict)
316    assert(expr.items.__class__ == list)
317    cf_dict = {}
318    for key, value in expr.items:
319        try:
320            key = _extract_const(key)
321            val = _extract_expression(value)
322        except (AssertionError, ValueError):
323            pass
324        else:
325            cf_dict[key] = val
326    return cf_dict
327
328
329def _extract_list(expr):
330    assert(expr.__class__ == compiler.ast.List)
331    list_values = []
332    for value in expr.nodes:
333        try:
334            list_values.append(_extract_expression(value))
335        except (AssertionError, ValueError):
336            pass
337    return list_values
338
339
340def _extract_name(expr):
341    assert(expr.__class__ == compiler.ast.Name)
342    assert(expr.name in ('False', 'True', 'None'))
343    return str(expr.name)
344
345
346def _extract_expression(expr):
347    if expr.__class__ == compiler.ast.Const:
348        return _extract_const(expr)
349    if expr.__class__ == compiler.ast.Name:
350        return _extract_name(expr)
351    if expr.__class__ == compiler.ast.Dict:
352        return _extract_dict(expr)
353    if expr.__class__ == compiler.ast.List:
354        return _extract_list(expr)
355    raise ValueError('Unknown rval %s' % expr)
356
357
358def _extract_assignment(n):
359    assert(n.__class__ == compiler.ast.Assign)
360    assert(n.nodes.__class__ == list)
361    assert(len(n.nodes) == 1)
362    assert(n.nodes[0].__class__ == compiler.ast.AssName)
363    assert(n.nodes[0].flags.__class__ == str)
364    assert(n.nodes[0].name.__class__ == str)
365
366    val = _extract_expression(n.expr)
367    key = n.nodes[0].name.lower()
368
369    return (key, val)
370
371
372def parse_control_string(control, raise_warnings=False, path=''):
373    """Parse a control file from a string.
374
375    @param control: string containing the text of a control file.
376    @param raise_warnings: True iff ControlData should raise an error on
377            warnings about control file contents.
378    @param path: string path to the control file.
379
380    """
381    try:
382        mod = compiler.parse(control)
383    except SyntaxError as e:
384        logging.error('Syntax error (%s) while parsing control string:', e)
385        lines = control.split('\n')
386        for n, l in enumerate(lines):
387            logging.error('Line %d: %s', n + 1, l)
388        raise ControlVariableException("Error parsing data because %s" % e)
389    return finish_parse(mod, path, raise_warnings)
390
391
392def parse_control(path, raise_warnings=False):
393    try:
394        mod = compiler.parseFile(path)
395    except SyntaxError, e:
396        raise ControlVariableException("Error parsing %s because %s" %
397                                       (path, e))
398    return finish_parse(mod, path, raise_warnings)
399
400
401def _try_extract_assignment(node, variables):
402    """Try to extract assignment from the given node.
403
404    @param node: An Assign object.
405    @param variables: Dictionary to store the parsed assignments.
406    """
407    try:
408        key, val = _extract_assignment(node)
409        variables[key] = val
410    except (AssertionError, ValueError):
411        pass
412
413
414def finish_parse(mod, path, raise_warnings):
415    assert(mod.__class__ == compiler.ast.Module)
416    assert(mod.node.__class__ == compiler.ast.Stmt)
417    assert(mod.node.nodes.__class__ == list)
418
419    variables = {}
420    injection_variables = {}
421    for n in mod.node.nodes:
422        if (n.__class__ == compiler.ast.Function and
423            re.match('step\d+', n.name)):
424            vars_in_step = {}
425            for sub_node in n.code.nodes:
426                _try_extract_assignment(sub_node, vars_in_step)
427            if vars_in_step:
428                # Empty the vars collection so assignments from multiple steps
429                # won't be mixed.
430                variables.clear()
431                variables.update(vars_in_step)
432        else:
433            _try_extract_assignment(n, injection_variables)
434
435    variables.update(injection_variables)
436    return ControlData(variables, path, raise_warnings)
437