• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5# repohooks/pre-upload.py currently does not run pylint. But for developers who
6# want to check their code manually we disable several harmless pylint warnings
7# which just distract from more serious remaining issues.
8#
9# The instance variable _android_cts is not defined in __init__().
10# pylint: disable=attribute-defined-outside-init
11#
12# Many short variable names don't follow the naming convention.
13# pylint: disable=invalid-name
14
15import logging
16import os
17
18from autotest_lib.server import utils
19from autotest_lib.server.cros import tradefed_test
20
21# Maximum default time allowed for each individual CTS module.
22_CTS_TIMEOUT_SECONDS = 3600
23
24# Public download locations for android cts bundles.
25_DL_CTS = 'https://dl.google.com/dl/android/cts/'
26_CTS_URI = {
27    'arm': _DL_CTS + 'android-cts-7.1_r14-linux_x86-arm.zip',
28    'x86': _DL_CTS + 'android-cts-7.1_r14-linux_x86-x86.zip',
29    'media': _DL_CTS + 'android-cts-media-1.4.zip',
30}
31
32
33class cheets_CTS_N(tradefed_test.TradefedTest):
34    """Sets up tradefed to run CTS tests."""
35    version = 1
36
37    # TODO(bmgordon): Remove kahlee once the bulk of failing tests are fixed.
38    _BOARD_RETRY = {'betty': 0, 'kahlee': 0}
39    _CHANNEL_RETRY = {'dev': 5, 'beta': 5, 'stable': 5}
40
41    def _get_default_bundle_url(self, bundle):
42        return _CTS_URI[bundle]
43
44    def _get_tradefed_base_dir(self):
45        return 'android-cts'
46
47    def _tradefed_run_command(self,
48                              module=None,
49                              plan=None,
50                              session_id=None,
51                              test_class=None,
52                              test_method=None):
53        """Builds the CTS tradefed 'run' command line.
54
55        There are five different usages:
56
57        1. Test a module: assign the module name via |module|.
58        2. Test a plan: assign the plan name via |plan|.
59        3. Continue a session: assign the session ID via |session_id|.
60        4. Run all test cases of a class: assign the class name via
61           |test_class|.
62        5. Run a specific test case: assign the class name and method name in
63           |test_class| and |test_method|.
64
65        @param module: the name of test module to be run.
66        @param plan: name of the plan to be run.
67        @param session_id: tradefed session id to continue.
68        @param test_class: the name of the class of which all test cases will
69                           be run.
70        @param test_name: the name of the method of |test_class| to be run.
71                          Must be used with |test_class|.
72        @return: list of command tokens for the 'run' command.
73        """
74        if module is not None:
75            # Run a particular module (used to be called package in M).
76            cmd = ['run', 'commandAndExit', 'cts', '--module', module]
77            if test_class is not None:
78                if test_method is not None:
79                    cmd += ['-t', test_class + '#' + test_method]
80                else:
81                    cmd += ['-t', test_class]
82        elif plan is not None and session_id is not None:
83            # In 7.1 R2 we can only retry session_id with the original plan.
84            cmd = ['run', 'commandAndExit', 'cts', '--plan', plan,
85                   '--retry', '%d' % session_id]
86        elif plan is not None:
87            # Subplan for any customized CTS test plan in form of xml.
88            cmd = ['run', 'commandAndExit', 'cts', '--subplan', plan]
89        else:
90            logging.warning('Running all tests. This can take several days.')
91            cmd = ['run', 'commandAndExit', 'cts']
92        # We handle media download ourselves in the lab, as lazy as possible.
93        cmd.append('--precondition-arg')
94        cmd.append('skip-media-download')
95        # If we are running outside of the lab we can collect more data.
96        if not utils.is_in_container():
97            logging.info('Running outside of lab, adding extra debug options.')
98            cmd.append('--log-level-display=DEBUG')
99            cmd.append('--screenshot-on-failure')
100            # TODO(ihf): Add log collection once b/28333587 fixed.
101            #cmd.append('--collect-deqp-logs')
102        # TODO(ihf): Add tradefed_test.adb_keepalive() and remove
103        # --disable-reboot. This might be more efficient.
104        # At early stage, cts-tradefed tries to reboot the device by
105        # "adb reboot" command. In a real Android device case, when the
106        # rebooting is completed, adb connection is re-established
107        # automatically, and cts-tradefed expects that behavior.
108        # However, in ARC, it doesn't work, so the whole test process
109        # is just stuck. Here, disable the feature.
110        cmd.append('--disable-reboot')
111        # Create a logcat file for each individual failure.
112        cmd.append('--logcat-on-failure')
113        return cmd
114
115    def _get_timeout_factor(self):
116        """Returns the factor to be multiplied to the timeout parameter.
117        The factor is determined by the number of ABIs to run."""
118        if self._timeoutfactor is None:
119            abilist = self._run('adb', args=('shell', 'getprop',
120                'ro.product.cpu.abilist')).stdout.split(',')
121            prefix = {'x86': 'x86', 'arm': 'armeabi-'}.get(self._abi)
122            self._timeoutfactor = (1 if prefix is None else
123                sum(1 for abi in abilist if abi.startswith(prefix)))
124        return self._timeoutfactor
125
126    def _run_tradefed(self, commands):
127        """Kick off CTS.
128
129        @param commands: the command(s) to pass to CTS.
130        @param datetime_id: For 'continue' datetime of previous run is known.
131        @return: The result object from utils.run.
132        """
133        cts_tradefed = os.path.join(self._repository, 'tools', 'cts-tradefed')
134        with tradefed_test.adb_keepalive(self._get_adb_target(),
135                                         self._install_paths):
136            for command in commands:
137                logging.info('RUN: ./cts-tradefed %s', ' '.join(command))
138                output = self._run(
139                    cts_tradefed,
140                    args=tuple(command),
141                    timeout=self._timeout * self._get_timeout_factor(),
142                    verbose=True,
143                    ignore_status=False,
144                    # Make sure to tee tradefed stdout/stderr to autotest logs
145                    # continuously during the test run.
146                    stdout_tee=utils.TEE_TO_LOGS,
147                    stderr_tee=utils.TEE_TO_LOGS)
148            logging.info('END: ./cts-tradefed %s\n', ' '.join(command))
149        return output
150
151    def _should_skip_test(self):
152        """Some tests are expected to fail and are skipped."""
153        # newbie and novato are x86 VMs without binary translation. Skip the ARM
154        # tests.
155        no_ARM_ABI_test_boards = ('newbie', 'novato', 'novato-arc64')
156        if self._get_board_name(self._host) in no_ARM_ABI_test_boards:
157            if self._abi == 'arm':
158                return True
159        return False
160
161    def generate_test_command(self, target_module, target_plan, target_class,
162                              target_method, tradefed_args, session_id=0):
163        """Generates the CTS command and name to use based on test arguments.
164
165        @param target_module: the name of test module to run.
166        @param target_plan: the name of the test plan to run.
167        @param target_class: the name of the class to be tested.
168        @param target_method: the name of the method to be tested.
169        @param tradefed_args: a list of args to pass to tradefed.
170        @param session_id: tradefed session_id.
171        """
172        if target_module is not None:
173            if target_class is not None:
174                test_name = 'testcase.%s' % target_class
175                if target_method is not None:
176                    test_name += '.' + target_method
177                test_command = self._tradefed_run_command(
178                    module=target_module,
179                    test_class=target_class,
180                    test_method=target_method,
181                    session_id=session_id)
182            else:
183                test_name = 'module.%s' % target_module
184                test_command = self._tradefed_run_command(
185                    module=target_module, session_id=session_id)
186        elif target_plan is not None:
187            test_name = 'plan.%s' % target_plan
188            test_command = self._tradefed_run_command(plan=target_plan)
189        elif tradefed_args is not None:
190            test_name = 'run tradefed %s' % ' '.join(tradefed_args)
191            test_command = tradefed_args
192        else:
193            test_command = self._tradefed_run_command()
194            test_name = 'all_CTS'
195
196        logging.info('CTS command: %s', test_command)
197        return test_command, test_name
198
199    def run_once(self,
200                 target_module=None,
201                 target_plan=None,
202                 target_class=None,
203                 target_method=None,
204                 needs_push_media=False,
205                 tradefed_args=None,
206                 precondition_commands=[],
207                 login_precondition_commands=[],
208                 timeout=_CTS_TIMEOUT_SECONDS):
209        """Runs the specified CTS once, but with several retries.
210
211        There are four usages:
212        1. Test the whole module named |target_module|.
213        2. Test with a plan named |target_plan|.
214        3. Run all the test cases of class named |target_class|.
215        4. Run a specific test method named |target_method| of class
216           |target_class|.
217        5. Run an arbitrary tradefed command.
218
219        @param target_module: the name of test module to run.
220        @param target_plan: the name of the test plan to run.
221        @param target_class: the name of the class to be tested.
222        @param target_method: the name of the method to be tested.
223        @param needs_push_media: need to push test media streams.
224        @param timeout: time after which tradefed can be interrupted.
225        @param precondition_commands: a list of scripts to be run on the
226        dut before the test is run, the scripts must already be installed.
227        @param login_precondition_commands: a list of scripts to be run on the
228        dut before the log-in for the test is performed.
229        @param tradefed_args: a list of args to pass to tradefed.
230        """
231
232        # On dev and beta channels timeouts are sharp, lenient on stable.
233        self._timeout = timeout
234        if self._get_release_channel(self._host) == 'stable':
235            self._timeout += 3600
236        # Retries depend on channel.
237        self._timeoutfactor = None
238
239        test_command, test_name = self.generate_test_command(target_module,
240                                                             target_plan,
241                                                             target_class,
242                                                             target_method,
243                                                             tradefed_args)
244
245        self._run_tradefed_with_retries(target_module, test_command, test_name,
246                                        target_plan, needs_push_media, _CTS_URI,
247                                        login_precondition_commands,
248                                        precondition_commands)
249