• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""\
2Logic for control file generation.
3"""
4
5__author__ = 'showard@google.com (Steve Howard)'
6
7import re, os
8
9import common
10from autotest_lib.client.common_lib import error
11from autotest_lib.client.common_lib.cros import dev_server
12from autotest_lib.frontend.afe import model_logic
13from autotest_lib.server.cros.dynamic_suite import control_file_getter
14from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase
15import frontend.settings
16
17AUTOTEST_DIR = os.path.abspath(os.path.join(
18    os.path.dirname(frontend.settings.__file__), '..'))
19
20EMPTY_TEMPLATE = 'def step_init():\n'
21
22CLIENT_STEP_TEMPLATE = "    job.next_step('step%d')\n"
23SERVER_STEP_TEMPLATE = '    step%d()\n'
24
25
26def _read_control_file(test):
27    """Reads the test control file from local disk.
28
29    @param test The test name.
30
31    @return The test control file string.
32    """
33    control_file = open(os.path.join(AUTOTEST_DIR, test.path))
34    control_contents = control_file.read()
35    control_file.close()
36    return control_contents
37
38
39def _add_boilerplate_to_nested_steps(lines):
40    """Adds boilerplate magic.
41
42    @param lines The string of lines.
43
44    @returns The string lines.
45    """
46    # Look for a line that begins with 'def step_init():' while
47    # being flexible on spacing.  If it's found, this will be
48    # a nested set of steps, so add magic to make it work.
49    # See client/bin/job.py's step_engine for more info.
50    if re.search(r'^(.*\n)*def\s+step_init\s*\(\s*\)\s*:', lines):
51        lines += '\nreturn locals() '
52        lines += '# Boilerplate magic for nested sets of steps'
53    return lines
54
55
56def _format_step(item, lines):
57    """Format a line item.
58    @param item The item number.
59    @param lines The string of lines.
60
61    @returns The string lines.
62    """
63    lines = _indent_text(lines, '    ')
64    lines = 'def step%d():\n%s' % (item, lines)
65    return lines
66
67
68def _get_tests_stanza(tests, is_server, prepend=None, append=None,
69                     client_control_file='', test_source_build=None):
70    """ Constructs the control file test step code from a list of tests.
71
72    @param tests A sequence of test control files to run.
73    @param is_server bool, Is this a server side test?
74    @param prepend A list of steps to prepend to each client test.
75        Defaults to [].
76    @param append A list of steps to append to each client test.
77        Defaults to [].
78    @param client_control_file If specified, use this text as the body of a
79        final client control file to run after tests.  is_server must be False.
80    @param test_source_build: Build to be used to retrieve test code. Default
81                              to None.
82
83    @returns The control file test code to be run.
84    """
85    assert not (client_control_file and is_server)
86    if not prepend:
87        prepend = []
88    if not append:
89        append = []
90    if test_source_build:
91        raw_control_files = _get_test_control_files_by_build(
92                tests, test_source_build)
93    else:
94        raw_control_files = [_read_control_file(test) for test in tests]
95    if client_control_file:
96        # 'return locals()' is always appended in case the user forgot, it
97        # is necessary to allow for nested step engine execution to work.
98        raw_control_files.append(client_control_file + '\nreturn locals()')
99    raw_steps = prepend + [_add_boilerplate_to_nested_steps(step)
100                           for step in raw_control_files] + append
101    steps = [_format_step(index, step)
102             for index, step in enumerate(raw_steps)]
103    if is_server:
104        step_template = SERVER_STEP_TEMPLATE
105        footer = '\n\nstep_init()\n'
106    else:
107        step_template = CLIENT_STEP_TEMPLATE
108        footer = ''
109
110    header = ''.join(step_template % i for i in xrange(len(steps)))
111    return header + '\n' + '\n\n'.join(steps) + footer
112
113
114def _indent_text(text, indent):
115    """Indent given lines of python code avoiding indenting multiline
116    quoted content (only for triple " and ' quoting for now).
117
118    @param text The string of lines.
119    @param indent The indent string.
120
121    @return The indented string.
122    """
123    regex = re.compile('(\\\\*)("""|\'\'\')')
124
125    res = []
126    in_quote = None
127    for line in text.splitlines():
128        # if not within a multinline quote indent the line contents
129        if in_quote:
130            res.append(line)
131        else:
132            res.append(indent + line)
133
134        while line:
135            match = regex.search(line)
136            if match:
137                # for an even number of backslashes before the triple quote
138                if len(match.group(1)) % 2 == 0:
139                    if not in_quote:
140                        in_quote = match.group(2)[0]
141                    elif in_quote == match.group(2)[0]:
142                        # if we found a matching end triple quote
143                        in_quote = None
144                line = line[match.end():]
145            else:
146                break
147
148    return '\n'.join(res)
149
150
151def _get_profiler_commands(profilers, is_server, profile_only):
152    prepend, append = [], []
153    if profile_only is not None:
154        prepend.append("job.default_profile_only = %r" % profile_only)
155    for profiler in profilers:
156        prepend.append("job.profilers.add('%s')" % profiler.name)
157        append.append("job.profilers.delete('%s')" % profiler.name)
158    return prepend, append
159
160
161def _sanity_check_generate_control(is_server, client_control_file):
162    """
163    Sanity check some of the parameters to generate_control().
164
165    This exists as its own function so that site_control_file may call it as
166    well from its own generate_control().
167
168    @raises ValidationError if any of the parameters do not make sense.
169    """
170    if is_server and client_control_file:
171        raise model_logic.ValidationError(
172                {'tests' : 'You cannot run server tests at the same time '
173                 'as directly supplying a client-side control file.'})
174
175
176def generate_control(tests, is_server=False, profilers=(),
177                     client_control_file='', profile_only=None,
178                     test_source_build=None):
179    """
180    Generate a control file for a sequence of tests.
181
182    @param tests A sequence of test control files to run.
183    @param is_server bool, Is this a server control file rather than a client?
184    @param profilers A list of profiler objects to enable during the tests.
185    @param client_control_file Contents of a client control file to run as the
186            last test after everything in tests.  Requires is_server=False.
187    @param profile_only bool, should this control file run all tests in
188            profile_only mode by default
189    @param test_source_build: Build to be used to retrieve test code. Default
190                              to None.
191
192    @returns The control file text as a string.
193    """
194    _sanity_check_generate_control(is_server=is_server,
195                                   client_control_file=client_control_file)
196    control_file_text = EMPTY_TEMPLATE
197    prepend, append = _get_profiler_commands(profilers, is_server, profile_only)
198    control_file_text += _get_tests_stanza(tests, is_server, prepend, append,
199                                           client_control_file,
200                                           test_source_build)
201    return control_file_text
202
203
204def _get_test_control_files_by_build(tests, build, ignore_invalid_tests=False):
205    """Get the test control files that are available for the specified build.
206
207    @param tests A sequence of test objects to run.
208    @param build: unique name by which to refer to the image.
209    @param ignore_invalid_tests: flag on if unparsable tests are ignored.
210
211    @return: A sorted list of all tests that are in the build specified.
212    """
213    raw_control_files = []
214    # shortcut to avoid staging the image.
215    if not tests:
216        return raw_control_files
217
218    cfile_getter = _initialize_control_file_getter(build)
219    if SuiteBase.ENABLE_CONTROLS_IN_BATCH:
220        control_file_info_list = cfile_getter.get_suite_info()
221
222    for test in tests:
223        # Read and parse the control file
224        if SuiteBase.ENABLE_CONTROLS_IN_BATCH:
225            control_file = control_file_info_list[test.path]
226        else:
227            control_file = cfile_getter.get_control_file_contents(
228                    test.path)
229        raw_control_files.append(control_file)
230    return raw_control_files
231
232
233def _initialize_control_file_getter(build):
234    """Get the remote control file getter.
235
236    @param build: unique name by which to refer to a remote build image.
237
238    @return: A control file getter object.
239    """
240    # Stage the test artifacts.
241    try:
242        ds = dev_server.ImageServer.resolve(build)
243        ds_name = ds.hostname
244        build = ds.translate(build)
245    except dev_server.DevServerException as e:
246        raise ValueError('Could not resolve build %s: %s' %
247                         (build, e))
248
249    try:
250        ds.stage_artifacts(image=build, artifacts=['test_suites'])
251    except dev_server.DevServerException as e:
252        raise error.StageControlFileFailure(
253                'Failed to stage %s on %s: %s' % (build, ds_name, e))
254
255    # Collect the control files specified in this build
256    return control_file_getter.DevServerGetter.create(build, ds)
257