1# Copyright 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Utility to deploy and run result utils on a DUT. 6 7This module is the one imported by other Autotest code and run result 8throttling. Other modules in result_tools are designed to be copied to DUT and 9executed with command line. That's why other modules (except view.py and 10unittests) don't import the common module. 11""" 12 13import logging 14import os 15 16import common 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.common_lib import global_config 19from autotest_lib.client.common_lib import utils as client_utils 20 21try: 22 from chromite.lib import metrics 23except ImportError: 24 metrics = client_utils.metrics_mock 25 26 27CONFIG = global_config.global_config 28ENABLE_RESULT_THROTTLING = CONFIG.get_config_value( 29 'AUTOSERV', 'enable_result_throttling', type=bool, default=False) 30 31_THROTTLE_OPTION_FMT = '-m %s' 32_SUMMARY_CMD= '%s/result_tools/utils.py' 33_BUILD_DIR_SUMMARY_CMD = _SUMMARY_CMD + ' -p %s %s' 34_BUILD_DIR_SUMMARY_TIMEOUT = 120 35_FIND_DIR_SUMMARY_TIMEOUT = 10 36_CLEANUP_DIR_SUMMARY_CMD = _SUMMARY_CMD + ' -p %s -d' 37_CLEANUP_DIR_SUMMARY_TIMEOUT = 10 38 39# Default autotest directory on host 40DEFAULT_AUTOTEST_DIR = '/usr/local/autotest' 41 42# File patterns to be excluded from deploying to the dut. 43_EXCLUDES = ['*.pyc', '*unittest.py', 'common.py', '__init__.py', 'runner.py', 44 'view.py'] 45 46def _deploy_result_tools(host): 47 """Send result tools to the dut. 48 49 @param host: Host to run the result tools. 50 """ 51 with metrics.SecondsTimer( 52 'chromeos/autotest/job/send_result_tools_duration', 53 fields={'dut_host_name': host.hostname}) as fields: 54 try: 55 result = host.run('test -f %s' % 56 (_SUMMARY_CMD % DEFAULT_AUTOTEST_DIR), 57 timeout=_FIND_DIR_SUMMARY_TIMEOUT, 58 ignore_status=True) 59 if result.exit_status == 0: 60 logging.debug('result tools are already deployed to %s.', 61 host.hostname) 62 else: 63 logging.debug('Deploy result utilities to %s', host.hostname) 64 result_tools_dir = os.path.dirname(__file__) 65 host.send_file(result_tools_dir, DEFAULT_AUTOTEST_DIR, 66 excludes = _EXCLUDES) 67 fields['success'] = True 68 except error.AutotestHostRunError: 69 logging.debug('Failed to deploy result tools using `excludes`. Try ' 70 'again without `excludes`.') 71 host.send_file(result_tools_dir, DEFAULT_AUTOTEST_DIR) 72 fields['success'] = False 73 74 75def run_on_client(host, client_results_dir, cleanup_only=False): 76 """Run result utils on the given host. 77 78 @param host: Host to run the result utils. 79 @param client_results_dir: Path to the results directory on the client. 80 @param cleanup_only: True to delete all existing directory summary files in 81 the given directory. 82 @return: True: If the command runs on client without error. 83 False: If the command failed with error in result throttling. 84 """ 85 success = False 86 with metrics.SecondsTimer( 87 'chromeos/autotest/job/dir_summary_collection_duration', 88 fields={'dut_host_name': host.hostname}) as fields: 89 try: 90 _deploy_result_tools(host) 91 92 if cleanup_only: 93 logging.debug('Cleaning up directory summary in %s', 94 client_results_dir) 95 cmd = (_CLEANUP_DIR_SUMMARY_CMD % 96 (DEFAULT_AUTOTEST_DIR, client_results_dir)) 97 host.run(cmd, ignore_status=False, 98 timeout=_CLEANUP_DIR_SUMMARY_TIMEOUT) 99 else: 100 logging.debug('Getting directory summary for %s', 101 client_results_dir) 102 throttle_option = '' 103 if ENABLE_RESULT_THROTTLING: 104 try: 105 throttle_option = (_THROTTLE_OPTION_FMT % 106 host.job.max_result_size_KB) 107 except AttributeError: 108 # In case host job is not set, skip throttling. 109 logging.warn('host object does not have job attribute, ' 110 'skipping result throttling.') 111 cmd = (_BUILD_DIR_SUMMARY_CMD % 112 (DEFAULT_AUTOTEST_DIR, client_results_dir, 113 throttle_option)) 114 host.run(cmd, ignore_status=False, 115 timeout=_BUILD_DIR_SUMMARY_TIMEOUT) 116 success = True 117 fields['success'] = True 118 except error.AutoservRunError: 119 action = 'cleanup' if cleanup_only else 'create' 120 logging.exception( 121 'Non-critical failure: Failed to %s directory summary for ' 122 '%s.', action, client_results_dir) 123 fields['success'] = False 124 125 return success 126 127 128def collect_last_summary(host, source_path, dest_path, 129 skip_summary_collection=False): 130 """Collect the last directory summary next to the given file path. 131 132 If the given source_path is a directory, return without collecting any 133 result summary file, as the summary file should have been collected with the 134 directory. 135 136 @param host: The RemoteHost to collect logs from. 137 @param source_path: The remote path to collect the directory summary file 138 from. If the source_path is a file 139 @param dest_path: A path to write the source_path into. The summary file 140 will be saved to the same folder. 141 @param skip_summary_collection: True to skip summary file collection, only 142 to delete the last summary. This is used in case when result 143 collection in the dut failed. Default is set to False. 144 """ 145 if not os.path.exists(dest_path): 146 logging.debug('Source path %s does not exist, no directory summary ' 147 'will be collected', dest_path) 148 return 149 150 # Test if source_path is a file. 151 try: 152 host.run('test -f %s' % source_path, timeout=_FIND_DIR_SUMMARY_TIMEOUT) 153 is_source_file = True 154 except error.AutoservRunError: 155 is_source_file = False 156 # No need to collect summary files if the source path is a directory, 157 # as the summary files should have been copied over with the directory. 158 # However, the last summary should be cleaned up so it won't affect 159 # later tests. 160 skip_summary_collection = True 161 162 source_dir = os.path.dirname(source_path) if is_source_file else source_path 163 dest_dir = dest_path if os.path.isdir(dest_path) else dest_path 164 165 # Get the latest directory summary file. 166 try: 167 summary_pattern = os.path.join(source_dir, 'dir_summary_*.json') 168 summary_file = host.run( 169 'ls -t %s | head -1' % summary_pattern, 170 timeout=_FIND_DIR_SUMMARY_TIMEOUT).stdout.strip() 171 except error.AutoservRunError: 172 logging.exception( 173 'Non-critical failure: Failed to locate the latest directory ' 174 'summary for %s', source_dir) 175 return 176 177 try: 178 if not skip_summary_collection: 179 host.get_file( 180 summary_file, 181 os.path.join(dest_dir, os.path.basename(summary_file)), 182 preserve_perm=False) 183 finally: 184 # Remove the collected summary file so it won't affect later tests. 185 try: 186 # Check and remove the summary file 187 if host.path_exists(summary_file): 188 host.run('rm %s' % summary_file, 189 timeout=_FIND_DIR_SUMMARY_TIMEOUT).stdout.strip() 190 except error.AutoservRunError: 191 logging.exception( 192 'Non-critical failure: Failed to delete the latest ' 193 'directory summary: %s', summary_file) 194