• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2018 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import logging
7import os
8
9from autotest_lib.client.common_lib import error
10from autotest_lib.server import test
11from autotest_lib.server import utils
12
13
14class tast_Runner(test.test):
15    """Autotest server test that runs a Tast test suite.
16
17    Tast is an integration-testing framework analagous to the test-running
18    portion of Autotest. See
19    https://chromium.googlesource.com/chromiumos/platform/tast/ for more
20    information.
21
22    This class runs the "tast" command locally to execute a Tast test suite on a
23    remote DUT.
24    """
25    version = 1
26
27    # Maximum time to wait for the tast command to complete, in seconds.
28    _EXEC_TIMEOUT_SEC = 600
29
30    # JSON file written by the tast command containing test results.
31    _RESULTS_FILENAME = 'results.json'
32
33    # Maximum number of failing tests to include in error message.
34    _MAX_TEST_NAMES_IN_ERROR = 3
35
36    # Default paths where Tast files are installed.
37    _TAST_PATH = '/usr/bin/tast'
38    _REMOTE_BUNDLE_DIR = '/usr/libexec/tast/bundles/remote'
39    _REMOTE_DATA_DIR = '/usr/share/tast/data/remote'
40    _REMOTE_TEST_RUNNER_PATH = '/usr/bin/remote_test_runner'
41
42    # When Tast is deployed from CIPD packages in the lab, it's installed under
43    # this prefix rather than under the root directory.
44    _CIPD_INSTALL_ROOT = '/opt/infra-tools'
45
46    def initialize(self, host, test_exprs):
47        """
48        @param host: remote.RemoteHost instance representing DUT.
49        @param test_exprs: Array of strings describing tests to run.
50
51        @raises error.TestFail if the Tast installation couldn't be found.
52        """
53        self._host = host
54        self._test_exprs = test_exprs
55
56        # The data dir can be missing if no remote tests registered data files,
57        # but all other files must exist.
58        self._tast_path = self._get_path(self._TAST_PATH)
59        self._remote_bundle_dir = self._get_path(self._REMOTE_BUNDLE_DIR)
60        self._remote_data_dir = self._get_path(self._REMOTE_DATA_DIR,
61                                               allow_missing=True)
62        self._remote_test_runner_path = self._get_path(
63                self._REMOTE_TEST_RUNNER_PATH)
64
65    def run_once(self):
66        """Runs the test suite once."""
67        self._log_version()
68        self._run_tast()
69        self._parse_results()
70
71    def _get_path(self, path, allow_missing=False):
72        """Returns the path to an installed Tast-related file or directory.
73
74        @param path Absolute path in root filesystem, e.g. "/usr/bin/tast".
75        @param allow_missing True if it's okay for the path to be missing.
76
77        @return: Absolute path within install root, e.g.
78            "/opt/infra-tools/usr/bin/tast", or an empty string if the path
79            wasn't found and allow_missing is True.
80
81        @raises error.TestFail if the path couldn't be found and allow_missing
82            is False.
83        """
84        if os.path.exists(path):
85            return path
86
87        cipd_path = os.path.join(self._CIPD_INSTALL_ROOT,
88                                 os.path.relpath(path, '/'))
89        if os.path.exists(cipd_path):
90            return cipd_path
91
92        if allow_missing:
93            return ''
94        raise error.TestFail('Neither %s nor %s exists' % (path, cipd_path))
95
96    def _log_version(self):
97        """Runs the tast command locally to log its version."""
98        try:
99            utils.run([self._tast_path, '-version'],
100                      timeout=self._EXEC_TIMEOUT_SEC,
101                      stdout_tee=utils.TEE_TO_LOGS,
102                      stderr_tee=utils.TEE_TO_LOGS,
103                      stderr_is_expected=True,
104                      stdout_level=logging.INFO,
105                      stderr_level=logging.ERROR)
106        except error.CmdError as e:
107            logging.error('Failed to log tast version: %s' % str(e))
108
109    def _run_tast(self):
110        """Runs the tast command locally to perform testing against the DUT.
111
112        @raises error.TestFail if the tast command fails or times out (but not
113            if individual tests fail).
114        """
115        cmd = [
116            self._tast_path,
117            '-verbose',
118            '-logtime=false',
119            'run',
120            '-build=false',
121            '-resultsdir=' + self.resultsdir,
122            '-remotebundledir=' + self._remote_bundle_dir,
123            '-remotedatadir=' + self._remote_data_dir,
124            '-remoterunner=' + self._remote_test_runner_path,
125            self._host.hostname,
126        ]
127        cmd.extend(self._test_exprs)
128
129        logging.info('Running ' +
130                     ' '.join([utils.sh_quote_word(a) for a in cmd]))
131        try:
132            utils.run(cmd,
133                      ignore_status=False,
134                      timeout=self._EXEC_TIMEOUT_SEC,
135                      stdout_tee=utils.TEE_TO_LOGS,
136                      stderr_tee=utils.TEE_TO_LOGS,
137                      stderr_is_expected=True,
138                      stdout_level=logging.INFO,
139                      stderr_level=logging.ERROR)
140        except error.CmdError as e:
141            raise error.TestFail('Failed to run tast: %s' % str(e))
142        except error.CmdTimeoutError as e:
143            raise error.TestFail('Got timeout while running tast: %s' % str(e))
144
145    def _parse_results(self):
146        """Parses results written by the tast command.
147
148        @raises error.TestFail if one or more tests failed.
149        """
150        path = os.path.join(self.resultsdir, self._RESULTS_FILENAME)
151        failed = []
152        with open(path, 'r') as f:
153            for test in json.load(f):
154                if test['errors']:
155                    name = test['name']
156                    for err in test['errors']:
157                        logging.warning('%s: %s', name, err['reason'])
158                    # TODO(derat): Report failures in flaky tests in some other
159                    # way.
160                    if 'flaky' not in test.get('attr', []):
161                        failed.append(name)
162
163        if failed:
164            msg = '%d failed: ' % len(failed)
165            msg += ' '.join(sorted(failed)[:self._MAX_TEST_NAMES_IN_ERROR])
166            if len(failed) > self._MAX_TEST_NAMES_IN_ERROR:
167                msg += ' ...'
168            raise error.TestFail(msg)
169