• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Help functions used by different throttlers."""
6
7import os
8import re
9
10import utils_lib
11
12
13# A list of file names that should not be throttled, that is, not modified by
14# deletion, trimming or compression.
15NON_THROTTLEABLE_FILE_NAMES = set([
16        '.autoserv_execute',
17        '.parse.lock',
18        '.parse.log',
19        '.parser_execute',
20        'control',
21        'control.srv',
22        'host_keyvals',
23        'job_report.html',
24        'keyval',
25        'profiling',
26        'result_summary.html',
27        'sponge_invocation.xml',
28        'status',
29        'status.log',
30
31        # ACTS related files:
32        'test_run_details.txt',
33        'test_run_error.txt',
34        'test_run_info.txt',
35        'test_run_summary.json',
36        ])
37
38# A list of file name patterns that should not be throttled, that is, not
39# modified by deletion, deduping, trimming or compression.
40NON_THROTTLEABLE_FILE_PATTERNS = [
41        '.*/BUILD_INFO-.*',   # ACTS test result files.
42        '.*/AndroidDevice.*', # ACTS test result files.
43        ]
44
45# Regex of result files sorted based on priority. Files can be throttled first
46# have higher priority.
47RESULT_THROTTLE_PRIORITY = [
48        '(.*/)?sysinfo/var/log/.*',
49        '(.*/)?sysinfo/var/log_diff/.*',
50        '(.*/)?sysinfo/.*',
51        # The last one matches any file.
52        '.*',
53        ]
54
55# Regex of file names for Autotest debug logs. These files should be preserved
56# without throttling if possible.
57AUTOTEST_LOG_PATTERN ='.*\.(DEBUG|ERROR|INFO|WARNING)$'
58
59def _list_files(files, all_files=None):
60    """Get all files in the given directories.
61
62    @param files: A list of ResultInfo objects for files in a directory.
63    @param all_files: A list of ResultInfo objects collected so far.
64    @return: A list of all collected ResultInfo objects.
65    """
66    if all_files is None:
67        all_files = []
68    for info in files:
69        if info.is_dir:
70            _list_files(info.files, all_files)
71        else:
72            all_files.append(info)
73    return all_files
74
75
76def sort_result_files(summary):
77    """Sort result infos based on priority.
78
79    @param summary: A ResultInfo object containing result summary.
80    @return: A tuple of (sorted_files, grouped_files)
81            sorted_files: A list of ResultInfo, sorted based on file size and
82                priority based on RESULT_THROTTLE_PRIORITY.
83            grouped_files: A dictionary of ResultInfo grouped by each item in
84                RESULT_THROTTLE_PRIORITY.
85    """
86    all_files = _list_files(summary.files)
87
88    # Scan all file paths and group them based on the throttle priority.
89    grouped_files = {pattern: [] for pattern in RESULT_THROTTLE_PRIORITY}
90    for info in all_files:
91        for pattern in RESULT_THROTTLE_PRIORITY:
92            if re.match(pattern, info.path):
93                grouped_files[pattern].append(info)
94                break
95
96    sorted_files = []
97    for pattern in RESULT_THROTTLE_PRIORITY:
98        # Sort the files in each group by file size, largest first.
99        infos = grouped_files[pattern]
100        infos.sort(key=lambda info: -info.trimmed_size)
101        sorted_files.extend(infos)
102
103    return sorted_files, grouped_files
104
105
106def get_throttleable_files(file_infos, extra_patterns=[]):
107    """Filter the files can be throttled.
108
109    @param file_infos: A list of ResultInfo objects.
110    @param extra_patterns: Extra patterns of file path that should not be
111            throttled.
112    @yield: ResultInfo objects that can be throttled.
113    """
114    for info in file_infos:
115        # Skip the files being deleted in earlier throttling.
116        if info.trimmed_size == 0:
117            continue
118        if info.name in NON_THROTTLEABLE_FILE_NAMES:
119            continue
120        pattern_matched = False
121        for pattern in extra_patterns + NON_THROTTLEABLE_FILE_PATTERNS:
122            if re.match(pattern, info.path):
123                pattern_matched = True
124                break
125
126        if not pattern_matched:
127            yield info
128
129
130def check_throttle_limit(summary, max_result_size_KB):
131    """Check if throttling is enough already.
132
133    @param summary: A ResultInfo object containing result summary.
134    @param max_result_size_KB: Maximum test result size in KB.
135    @return: True if the result directory has been trimmed to be smaller than
136            max_result_size_KB.
137    """
138    if (summary.trimmed_size <= max_result_size_KB * 1024):
139        utils_lib.LOG('Maximum result size is reached (current result'
140                      'size is %s (limit is %s).' %
141                      (utils_lib.get_size_string(summary.trimmed_size),
142                       utils_lib.get_size_string(max_result_size_KB * 1024)))
143        return True
144    else:
145        return False
146
147
148def try_delete_file_on_disk(path):
149    """Try to delete the give file on disk.
150
151    @param path: Path to the file.
152    @returns: True if the file is deleted, False otherwise.
153    """
154    try:
155        utils_lib.LOG('Deleting file %s.' % path)
156        os.remove(path)
157        return True
158    except OSError as e:
159        utils_lib.LOG('Failed to delete file %s, Error: %s' % (path, e))