• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Lint as: python2, python3
2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import errno
7import logging
8import os
9import shutil
10from six.moves import urllib
11import subprocess
12
13from autotest_lib.client.common_lib import global_config
14
15def rm_dir_if_exists(dir_to_remove):
16    """
17    Removes a directory. Does not fail if the directory does NOT exist.
18
19    @param dir_to_remove: path, directory to be removed.
20
21    """
22    try:
23        shutil.rmtree(dir_to_remove)
24    except OSError as e:
25        if e.errno != errno.ENOENT:
26            raise
27
28
29def rm_dirs_if_exist(dirs_to_remove):
30    """
31    Removes multiple directories. Does not fail if directories do NOT exist.
32
33    @param dirs_to_remove: list of directory paths to be removed.
34
35    """
36    for dr in dirs_to_remove:
37        rm_dir_if_exists(dr)
38
39
40def ensure_file_exists(filepath):
41    """
42    Verifies path given points to an existing file.
43
44    @param filepath: path, path to check.
45
46    @raises IOError if the path given does not point to a valid file.
47
48    """
49    error_msg = 'File %s does not exist.' % filepath
50    if not os.path.isfile(filepath):
51        raise IOError(error_msg)
52
53
54def ensure_all_files_exist(filepaths):
55    """
56    Verifies all paths given point to existing files.
57
58    @param filepaths: List of paths to check.
59
60    @raises IOError if given paths do not point to existing files.
61
62    """
63    for filepath in filepaths:
64        ensure_file_exists(filepath)
65
66
67def ensure_dir_exists(dirpath):
68    """
69    Verifies path given points to an existing directory.
70
71    @param dirpath: path, dir to check.
72
73    @raises IOError if path does not point to an existing directory.
74
75    """
76    error_msg = 'Directory %s does not exist.' % dirpath
77    if not os.path.isdir(dirpath):
78        raise IOError(error_msg)
79
80
81def ensure_all_dirs_exist(dirpaths):
82    """
83    Verifies all paths given point to existing directories.
84
85    @param dirpaths: list of directory paths to check.
86
87    @raises IOError if given paths do not point to existing directories.
88
89    """
90    for dirpath in dirpaths:
91        ensure_dir_exists(dirpath)
92
93
94def make_leaf_dir(dirpath):
95    """
96    Creates a directory, also creating parent directories if they do not exist.
97
98    @param dirpath: path, directory to create.
99
100    @raises whatever exception raised other than "path already exist".
101
102    """
103    try:
104        os.makedirs(dirpath)
105    except OSError as e:
106        if e.errno != errno.EEXIST:
107            raise
108
109
110def make_leaf_dirs(dirpaths):
111    """
112    Creates multiple directories building all respective parent directories if
113    they do not exist.
114
115    @param dirpaths: list of directory paths to create.
116
117    @raises whatever exception raised other than "path already exists".
118    """
119    for dirpath in dirpaths:
120        make_leaf_dir(dirpath)
121
122
123def download_file(remote_path, local_path):
124    """
125    Download file from a remote resource.
126
127    @param remote_path: path, complete path to the remote file.
128    @param local_path: path, complete path to save downloaded file.
129
130    @raises: urllib2.HTTPError or urlib2.URLError exception. Both with added
131            debug information
132
133    """
134    client_config = global_config.global_config.get_section_values('CLIENT')
135    proxies = {}
136
137    for name, value in client_config.items('CLIENT'):
138        if value and name.endswith('_proxy'):
139            proxies[name[:-6]] = value
140
141    if proxies:
142        proxy_handler = urllib.request.ProxyHandler(proxies)
143        opener = urllib.request.build_opener(proxy_handler)
144        urllib.request.install_opener(opener)
145
146    # Unlike urllib.urlopen urllib2.urlopen will immediately throw on error
147    # If we could not find the file pointed by remote_path we will get an
148    # exception, catch the exception to log useful information then re-raise
149
150    try:
151        remote_file = urllib.request.urlopen(remote_path)
152
153        # Catch exceptions, extract exception properties and then re-raise
154        # This helps us with debugging what went wrong quickly as we get to see
155        # test_that output immediately
156
157    except urllib.error.HTTPError as e:
158        e.msg = (("""HTTPError raised while retrieving file %s\n.
159                       Http Code = %s.\n. Reason = %s\n. Headers = %s.\n
160                       Original Message = %s.\n""")
161                 % (remote_path, e.code, e.reason, e.headers, e.msg))
162        raise
163
164    except urllib.error.URLError as e:
165        e.msg = (("""URLError raised while retrieving file %s\n.
166                        Reason = %s\n. Original Message = %s\n.""")
167                 % (remote_path, e.reason, str(e)))
168        raise
169
170    with open(local_path, 'wb') as local_file:
171        while True:
172            block = remote_file.read(128 * 1024)
173            if not block:
174                break
175            local_file.write(block)
176
177
178def get_directory_size_kibibytes(directory):
179    """Calculate the total size of a directory with all its contents.
180
181    @param directory: Path to the directory
182
183    @return Size of the directory in kibibytes.
184    """
185    cmd = ['du', '-sk', directory]
186    process = subprocess.Popen(cmd,
187                               stdout=subprocess.PIPE,
188                               stderr=subprocess.PIPE)
189    stdout_data, stderr_data = process.communicate()
190
191    if process.returncode != 0:
192        # This function is used for statistics only, if it fails,
193        # nothing else should crash.
194        logging.warning('Getting size of %s failed. Stderr:', directory)
195        logging.warning(stderr_data)
196        return 0
197
198    return int(stdout_data.split(b'\t', 1)[0])
199
200
201def recursive_path_permission(path):
202    """
203    Recursively lists a path and its parent's permission.
204
205    For a path, it emits a path_perm string "<path> <permission>" where
206    <permission> is an octal representation of the path's st_mode. After that,
207    it emits its parent's path_perm string(s) recursively.
208    For example, recursive_path_permission('/foo/bar/buz') returns
209    ['/foo/bar/buz 0100644', '/foo/bar 040755', '/foo 040755', '/ 040755']
210
211    If |path| is invalid, it returns empty list.
212    If |path| does not have parent directory, it returns a list with only its
213    own path_perm string.
214
215    @param path: file path.
216
217    @return list of "<path> <permission>" string.
218    """
219    def path_st_mode(p):
220        """
221        Gets path's permission.
222
223        @param p: file path.
224
225        @return "<path> <permission>" where <permission> is an octal
226        representation of the path's st_mode.
227        """
228        return '%s %s' % (p, oct(os.stat(p).st_mode))
229
230    if not path or not os.path.exists(path):
231        return []
232    if path == '/':
233        return [path_st_mode(path)]
234    dirname, basename = os.path.split(path)
235    if not basename:
236        return recursive_path_permission(dirname)
237    result = [path_st_mode(path)]
238    if dirname:
239        result.extend(recursive_path_permission(dirname))
240    return result
241