1# Copyright 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Help functions used by different throttlers.""" 6 7import os 8import re 9 10import utils_lib 11 12 13# A list of file names that should not be throttled, that is, not modified by 14# deletion, trimming or compression. 15NON_THROTTLEABLE_FILE_NAMES = set([ 16 '.autoserv_execute', 17 '.parse.lock', 18 '.parse.log', 19 '.parser_execute', 20 'control', 21 'control.srv', 22 'host_keyvals', 23 'job_report.html', 24 'keyval', 25 'profiling', 26 'result_summary.html', 27 'sponge_invocation.xml', 28 'status', 29 'status.log', 30 31 # ACTS related files: 32 'test_run_details.txt', 33 'test_run_error.txt', 34 'test_run_info.txt', 35 'test_run_summary.json', 36 ]) 37 38# A list of file name patterns that should not be throttled, that is, not 39# modified by deletion, deduping, trimming or compression. 40NON_THROTTLEABLE_FILE_PATTERNS = [ 41 '.*/BUILD_INFO-.*', # ACTS test result files. 42 '.*/AndroidDevice.*', # ACTS test result files. 43 ] 44 45# Regex of result files sorted based on priority. Files can be throttled first 46# have higher priority. 47RESULT_THROTTLE_PRIORITY = [ 48 '(.*/)?sysinfo/var/log/.*', 49 '(.*/)?sysinfo/var/log_diff/.*', 50 '(.*/)?sysinfo/.*', 51 # The last one matches any file. 52 '.*', 53 ] 54 55# Regex of file names for Autotest debug logs. These files should be preserved 56# without throttling if possible. 57AUTOTEST_LOG_PATTERN ='.*\.(DEBUG|ERROR|INFO|WARNING)$' 58 59def _list_files(files, all_files=None): 60 """Get all files in the given directories. 61 62 @param files: A list of ResultInfo objects for files in a directory. 63 @param all_files: A list of ResultInfo objects collected so far. 64 @return: A list of all collected ResultInfo objects. 65 """ 66 if all_files is None: 67 all_files = [] 68 for info in files: 69 if info.is_dir: 70 _list_files(info.files, all_files) 71 else: 72 all_files.append(info) 73 return all_files 74 75 76def sort_result_files(summary): 77 """Sort result infos based on priority. 78 79 @param summary: A ResultInfo object containing result summary. 80 @return: A tuple of (sorted_files, grouped_files) 81 sorted_files: A list of ResultInfo, sorted based on file size and 82 priority based on RESULT_THROTTLE_PRIORITY. 83 grouped_files: A dictionary of ResultInfo grouped by each item in 84 RESULT_THROTTLE_PRIORITY. 85 """ 86 all_files = _list_files(summary.files) 87 88 # Scan all file paths and group them based on the throttle priority. 89 grouped_files = {pattern: [] for pattern in RESULT_THROTTLE_PRIORITY} 90 for info in all_files: 91 for pattern in RESULT_THROTTLE_PRIORITY: 92 if re.match(pattern, info.path): 93 grouped_files[pattern].append(info) 94 break 95 96 sorted_files = [] 97 for pattern in RESULT_THROTTLE_PRIORITY: 98 # Sort the files in each group by file size, largest first. 99 infos = grouped_files[pattern] 100 infos.sort(key=lambda info: -info.trimmed_size) 101 sorted_files.extend(infos) 102 103 return sorted_files, grouped_files 104 105 106def get_throttleable_files(file_infos, extra_patterns=[]): 107 """Filter the files can be throttled. 108 109 @param file_infos: A list of ResultInfo objects. 110 @param extra_patterns: Extra patterns of file path that should not be 111 throttled. 112 @yield: ResultInfo objects that can be throttled. 113 """ 114 for info in file_infos: 115 # Skip the files being deleted in earlier throttling. 116 if info.trimmed_size == 0: 117 continue 118 if info.name in NON_THROTTLEABLE_FILE_NAMES: 119 continue 120 pattern_matched = False 121 for pattern in extra_patterns + NON_THROTTLEABLE_FILE_PATTERNS: 122 if re.match(pattern, info.path): 123 pattern_matched = True 124 break 125 126 if not pattern_matched: 127 yield info 128 129 130def check_throttle_limit(summary, max_result_size_KB): 131 """Check if throttling is enough already. 132 133 @param summary: A ResultInfo object containing result summary. 134 @param max_result_size_KB: Maximum test result size in KB. 135 @return: True if the result directory has been trimmed to be smaller than 136 max_result_size_KB. 137 """ 138 if (summary.trimmed_size <= max_result_size_KB * 1024): 139 utils_lib.LOG('Maximum result size is reached (current result' 140 'size is %s (limit is %s).' % 141 (utils_lib.get_size_string(summary.trimmed_size), 142 utils_lib.get_size_string(max_result_size_KB * 1024))) 143 return True 144 else: 145 return False 146 147 148def try_delete_file_on_disk(path): 149 """Try to delete the give file on disk. 150 151 @param path: Path to the file. 152 @returns: True if the file is deleted, False otherwise. 153 """ 154 try: 155 utils_lib.LOG('Deleting file %s.' % path) 156 os.remove(path) 157 return True 158 except OSError as e: 159 utils_lib.LOG('Failed to delete file %s, Error: %s' % (path, e))