• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Utility to deploy and run result utils on a DUT.
6
7This module is the one imported by other Autotest code and run result
8throttling. Other modules in result_tools are designed to be copied to DUT and
9executed with command line. That's why other modules (except view.py and
10unittests) don't import the common module.
11"""
12
13import logging
14import os
15
16import common
17from autotest_lib.client.common_lib import error
18from autotest_lib.client.common_lib import global_config
19from autotest_lib.client.common_lib import utils as client_utils
20
21try:
22    from chromite.lib import metrics
23except ImportError:
24    metrics = client_utils.metrics_mock
25
26
27CONFIG = global_config.global_config
28ENABLE_RESULT_THROTTLING = CONFIG.get_config_value(
29        'AUTOSERV', 'enable_result_throttling', type=bool, default=False)
30
31_THROTTLE_OPTION_FMT = '-m %s'
32_SUMMARY_CMD= '%s/result_tools/utils.py'
33_BUILD_DIR_SUMMARY_CMD = _SUMMARY_CMD + ' -p %s %s'
34_BUILD_DIR_SUMMARY_TIMEOUT = 120
35_FIND_DIR_SUMMARY_TIMEOUT = 10
36_CLEANUP_DIR_SUMMARY_CMD = _SUMMARY_CMD + ' -p %s -d'
37_CLEANUP_DIR_SUMMARY_TIMEOUT = 10
38
39# Default autotest directory on host
40DEFAULT_AUTOTEST_DIR = '/usr/local/autotest'
41
42# File patterns to be excluded from deploying to the dut.
43_EXCLUDES = ['*.pyc', '*unittest.py', 'common.py', '__init__.py', 'runner.py',
44             'view.py']
45
46def _deploy_result_tools(host):
47    """Send result tools to the dut.
48
49    @param host: Host to run the result tools.
50    """
51    with metrics.SecondsTimer(
52            'chromeos/autotest/job/send_result_tools_duration',
53            fields={'dut_host_name': host.hostname}) as fields:
54        try:
55            result = host.run('test -f %s' %
56                      (_SUMMARY_CMD % DEFAULT_AUTOTEST_DIR),
57                   timeout=_FIND_DIR_SUMMARY_TIMEOUT,
58                   ignore_status=True)
59            if result.exit_status == 0:
60                logging.debug('result tools are already deployed to %s.',
61                        host.hostname)
62            else:
63                logging.debug('Deploy result utilities to %s', host.hostname)
64                result_tools_dir = os.path.dirname(__file__)
65                host.send_file(result_tools_dir, DEFAULT_AUTOTEST_DIR,
66                               excludes = _EXCLUDES)
67            fields['success'] = True
68        except error.AutotestHostRunError:
69            logging.debug('Failed to deploy result tools using `excludes`. Try '
70                          'again without `excludes`.')
71            host.send_file(result_tools_dir, DEFAULT_AUTOTEST_DIR)
72            fields['success'] = False
73
74
75def run_on_client(host, client_results_dir, cleanup_only=False):
76    """Run result utils on the given host.
77
78    @param host: Host to run the result utils.
79    @param client_results_dir: Path to the results directory on the client.
80    @param cleanup_only: True to delete all existing directory summary files in
81            the given directory.
82    @return: True: If the command runs on client without error.
83             False: If the command failed with error in result throttling.
84    """
85    success = False
86    with metrics.SecondsTimer(
87            'chromeos/autotest/job/dir_summary_collection_duration',
88            fields={'dut_host_name': host.hostname}) as fields:
89        try:
90            _deploy_result_tools(host)
91
92            if cleanup_only:
93                logging.debug('Cleaning up directory summary in %s',
94                              client_results_dir)
95                cmd = (_CLEANUP_DIR_SUMMARY_CMD %
96                       (DEFAULT_AUTOTEST_DIR, client_results_dir))
97                host.run(cmd, ignore_status=False,
98                         timeout=_CLEANUP_DIR_SUMMARY_TIMEOUT)
99            else:
100                logging.debug('Getting directory summary for %s',
101                              client_results_dir)
102                throttle_option = ''
103                if ENABLE_RESULT_THROTTLING:
104                    try:
105                        throttle_option = (_THROTTLE_OPTION_FMT %
106                                           host.job.max_result_size_KB)
107                    except AttributeError:
108                        # In case host job is not set, skip throttling.
109                        logging.warn('host object does not have job attribute, '
110                                     'skipping result throttling.')
111                cmd = (_BUILD_DIR_SUMMARY_CMD %
112                       (DEFAULT_AUTOTEST_DIR, client_results_dir,
113                        throttle_option))
114                host.run(cmd, ignore_status=False,
115                         timeout=_BUILD_DIR_SUMMARY_TIMEOUT)
116                success = True
117            fields['success'] = True
118        except error.AutoservRunError:
119            action = 'cleanup' if cleanup_only else 'create'
120            logging.exception(
121                    'Non-critical failure: Failed to %s directory summary for '
122                    '%s.', action, client_results_dir)
123            fields['success'] = False
124
125    return success
126
127
128def collect_last_summary(host, source_path, dest_path,
129                         skip_summary_collection=False):
130    """Collect the last directory summary next to the given file path.
131
132    If the given source_path is a directory, return without collecting any
133    result summary file, as the summary file should have been collected with the
134    directory.
135
136    @param host: The RemoteHost to collect logs from.
137    @param source_path: The remote path to collect the directory summary file
138            from. If the source_path is a file
139    @param dest_path: A path to write the source_path into. The summary file
140            will be saved to the same folder.
141    @param skip_summary_collection: True to skip summary file collection, only
142            to delete the last summary. This is used in case when result
143            collection in the dut failed. Default is set to False.
144    """
145    if not os.path.exists(dest_path):
146        logging.debug('Source path %s does not exist, no directory summary '
147                      'will be collected', dest_path)
148        return
149
150    # Test if source_path is a file.
151    try:
152        host.run('test -f %s' % source_path, timeout=_FIND_DIR_SUMMARY_TIMEOUT)
153        is_source_file = True
154    except error.AutoservRunError:
155        is_source_file = False
156        # No need to collect summary files if the source path is a directory,
157        # as the summary files should have been copied over with the directory.
158        # However, the last summary should be cleaned up so it won't affect
159        # later tests.
160        skip_summary_collection = True
161
162    source_dir = os.path.dirname(source_path) if is_source_file else source_path
163    dest_dir = dest_path if os.path.isdir(dest_path) else dest_path
164
165    # Get the latest directory summary file.
166    try:
167        summary_pattern = os.path.join(source_dir, 'dir_summary_*.json')
168        summary_file = host.run(
169                'ls -t %s | head -1' % summary_pattern,
170                timeout=_FIND_DIR_SUMMARY_TIMEOUT).stdout.strip()
171    except error.AutoservRunError:
172        logging.exception(
173                'Non-critical failure: Failed to locate the latest directory '
174                'summary for %s', source_dir)
175        return
176
177    try:
178        if not skip_summary_collection:
179            host.get_file(
180                    summary_file,
181                    os.path.join(dest_dir, os.path.basename(summary_file)),
182                    preserve_perm=False)
183    finally:
184        # Remove the collected summary file so it won't affect later tests.
185        try:
186            # Check and remove the summary file
187            if host.path_exists(summary_file):
188                host.run('rm %s' % summary_file,
189                         timeout=_FIND_DIR_SUMMARY_TIMEOUT).stdout.strip()
190        except error.AutoservRunError:
191            logging.exception(
192                    'Non-critical failure: Failed to delete the latest '
193                    'directory summary: %s', summary_file)
194