• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import ast
6import contextlib
7import fnmatch
8import json
9import os
10import pipes
11import re
12import shlex
13import shutil
14import subprocess
15import sys
16import tempfile
17import zipfile
18
19
20CHROMIUM_SRC = os.path.normpath(
21    os.path.join(os.path.dirname(__file__),
22                 os.pardir, os.pardir, os.pardir, os.pardir))
23COLORAMA_ROOT = os.path.join(CHROMIUM_SRC,
24                             'third_party', 'colorama', 'src')
25
26
27@contextlib.contextmanager
28def TempDir():
29  dirname = tempfile.mkdtemp()
30  try:
31    yield dirname
32  finally:
33    shutil.rmtree(dirname)
34
35
36def MakeDirectory(dir_path):
37  try:
38    os.makedirs(dir_path)
39  except OSError:
40    pass
41
42
43def DeleteDirectory(dir_path):
44  if os.path.exists(dir_path):
45    shutil.rmtree(dir_path)
46
47
48def Touch(path, fail_if_missing=False):
49  if fail_if_missing and not os.path.exists(path):
50    raise Exception(path + ' doesn\'t exist.')
51
52  MakeDirectory(os.path.dirname(path))
53  with open(path, 'a'):
54    os.utime(path, None)
55
56
57def FindInDirectory(directory, filename_filter):
58  files = []
59  for root, _dirnames, filenames in os.walk(directory):
60    matched_files = fnmatch.filter(filenames, filename_filter)
61    files.extend((os.path.join(root, f) for f in matched_files))
62  return files
63
64
65def FindInDirectories(directories, filename_filter):
66  all_files = []
67  for directory in directories:
68    all_files.extend(FindInDirectory(directory, filename_filter))
69  return all_files
70
71
72def ParseGnList(gn_string):
73  return ast.literal_eval(gn_string)
74
75
76def ParseGypList(gyp_string):
77  # The ninja generator doesn't support $ in strings, so use ## to
78  # represent $.
79  # TODO(cjhopman): Remove when
80  # https://code.google.com/p/gyp/issues/detail?id=327
81  # is addressed.
82  gyp_string = gyp_string.replace('##', '$')
83
84  if gyp_string.startswith('['):
85    return ParseGnList(gyp_string)
86  return shlex.split(gyp_string)
87
88
89def CheckOptions(options, parser, required=None):
90  if not required:
91    return
92  for option_name in required:
93    if getattr(options, option_name) is None:
94      parser.error('--%s is required' % option_name.replace('_', '-'))
95
96
97def WriteJson(obj, path, only_if_changed=False):
98  old_dump = None
99  if os.path.exists(path):
100    with open(path, 'r') as oldfile:
101      old_dump = oldfile.read()
102
103  new_dump = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '))
104
105  if not only_if_changed or old_dump != new_dump:
106    with open(path, 'w') as outfile:
107      outfile.write(new_dump)
108
109
110def ReadJson(path):
111  with open(path, 'r') as jsonfile:
112    return json.load(jsonfile)
113
114
115class CalledProcessError(Exception):
116  """This exception is raised when the process run by CheckOutput
117  exits with a non-zero exit code."""
118
119  def __init__(self, cwd, args, output):
120    super(CalledProcessError, self).__init__()
121    self.cwd = cwd
122    self.args = args
123    self.output = output
124
125  def __str__(self):
126    # A user should be able to simply copy and paste the command that failed
127    # into their shell.
128    copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd),
129        ' '.join(map(pipes.quote, self.args)))
130    return 'Command failed: {}\n{}'.format(copyable_command, self.output)
131
132
133# This can be used in most cases like subprocess.check_output(). The output,
134# particularly when the command fails, better highlights the command's failure.
135# If the command fails, raises a build_utils.CalledProcessError.
136def CheckOutput(args, cwd=None,
137                print_stdout=False, print_stderr=True,
138                stdout_filter=None,
139                stderr_filter=None,
140                fail_func=lambda returncode, stderr: returncode != 0):
141  if not cwd:
142    cwd = os.getcwd()
143
144  child = subprocess.Popen(args,
145      stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
146  stdout, stderr = child.communicate()
147
148  if stdout_filter is not None:
149    stdout = stdout_filter(stdout)
150
151  if stderr_filter is not None:
152    stderr = stderr_filter(stderr)
153
154  if fail_func(child.returncode, stderr):
155    raise CalledProcessError(cwd, args, stdout + stderr)
156
157  if print_stdout:
158    sys.stdout.write(stdout)
159  if print_stderr:
160    sys.stderr.write(stderr)
161
162  return stdout
163
164
165def GetModifiedTime(path):
166  # For a symlink, the modified time should be the greater of the link's
167  # modified time and the modified time of the target.
168  return max(os.lstat(path).st_mtime, os.stat(path).st_mtime)
169
170
171def IsTimeStale(output, inputs):
172  if not os.path.exists(output):
173    return True
174
175  output_time = GetModifiedTime(output)
176  for i in inputs:
177    if GetModifiedTime(i) > output_time:
178      return True
179  return False
180
181
182def IsDeviceReady():
183  device_state = CheckOutput(['adb', 'get-state'])
184  return device_state.strip() == 'device'
185
186
187def CheckZipPath(name):
188  if os.path.normpath(name) != name:
189    raise Exception('Non-canonical zip path: %s' % name)
190  if os.path.isabs(name):
191    raise Exception('Absolute zip path: %s' % name)
192
193
194def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None):
195  if path is None:
196    path = os.getcwd()
197  elif not os.path.exists(path):
198    MakeDirectory(path)
199
200  with zipfile.ZipFile(zip_path) as z:
201    for name in z.namelist():
202      if name.endswith('/'):
203        continue
204      if pattern is not None:
205        if not fnmatch.fnmatch(name, pattern):
206          continue
207      CheckZipPath(name)
208      if no_clobber:
209        output_path = os.path.join(path, name)
210        if os.path.exists(output_path):
211          raise Exception(
212              'Path already exists from zip: %s %s %s'
213              % (zip_path, name, output_path))
214
215    z.extractall(path=path)
216
217
218def DoZip(inputs, output, base_dir):
219  with zipfile.ZipFile(output, 'w') as outfile:
220    for f in inputs:
221      CheckZipPath(os.path.relpath(f, base_dir))
222      outfile.write(f, os.path.relpath(f, base_dir))
223
224
225def ZipDir(output, base_dir):
226  with zipfile.ZipFile(output, 'w') as outfile:
227    for root, _, files in os.walk(base_dir):
228      for f in files:
229        path = os.path.join(root, f)
230        archive_path = os.path.relpath(path, base_dir)
231        CheckZipPath(archive_path)
232        outfile.write(path, archive_path)
233
234
235def MergeZips(output, inputs, exclude_patterns=None):
236  def Allow(name):
237    if exclude_patterns is not None:
238      for p in exclude_patterns:
239        if fnmatch.fnmatch(name, p):
240          return False
241    return True
242
243  with zipfile.ZipFile(output, 'w') as out_zip:
244    for in_file in inputs:
245      with zipfile.ZipFile(in_file, 'r') as in_zip:
246        for name in in_zip.namelist():
247          if Allow(name):
248            out_zip.writestr(name, in_zip.read(name))
249
250
251def PrintWarning(message):
252  print 'WARNING: ' + message
253
254
255def PrintBigWarning(message):
256  print '*****     ' * 8
257  PrintWarning(message)
258  print '*****     ' * 8
259
260
261def GetSortedTransitiveDependencies(top, deps_func):
262  """Gets the list of all transitive dependencies in sorted order.
263
264  There should be no cycles in the dependency graph.
265
266  Args:
267    top: a list of the top level nodes
268    deps_func: A function that takes a node and returns its direct dependencies.
269  Returns:
270    A list of all transitive dependencies of nodes in top, in order (a node will
271    appear in the list at a higher index than all of its dependencies).
272  """
273  def Node(dep):
274    return (dep, deps_func(dep))
275
276  # First: find all deps
277  unchecked_deps = list(top)
278  all_deps = set(top)
279  while unchecked_deps:
280    dep = unchecked_deps.pop()
281    new_deps = deps_func(dep).difference(all_deps)
282    unchecked_deps.extend(new_deps)
283    all_deps = all_deps.union(new_deps)
284
285  # Then: simple, slow topological sort.
286  sorted_deps = []
287  unsorted_deps = dict(map(Node, all_deps))
288  while unsorted_deps:
289    for library, dependencies in unsorted_deps.items():
290      if not dependencies.intersection(unsorted_deps.keys()):
291        sorted_deps.append(library)
292        del unsorted_deps[library]
293
294  return sorted_deps
295
296
297def GetPythonDependencies():
298  """Gets the paths of imported non-system python modules.
299
300  A path is assumed to be a "system" import if it is outside of chromium's
301  src/. The paths will be relative to the current directory.
302  """
303  module_paths = (m.__file__ for m in sys.modules.itervalues()
304                  if m is not None and hasattr(m, '__file__'))
305
306  abs_module_paths = map(os.path.abspath, module_paths)
307
308  non_system_module_paths = [
309      p for p in abs_module_paths if p.startswith(CHROMIUM_SRC)]
310  def ConvertPycToPy(s):
311    if s.endswith('.pyc'):
312      return s[:-1]
313    return s
314
315  non_system_module_paths = map(ConvertPycToPy, non_system_module_paths)
316  non_system_module_paths = map(os.path.relpath, non_system_module_paths)
317  return sorted(set(non_system_module_paths))
318
319
320def AddDepfileOption(parser):
321  parser.add_option('--depfile',
322                    help='Path to depfile. This must be specified as the '
323                    'action\'s first output.')
324
325
326def WriteDepfile(path, dependencies):
327  with open(path, 'w') as depfile:
328    depfile.write(path)
329    depfile.write(': ')
330    depfile.write(' '.join(dependencies))
331    depfile.write('\n')
332
333
334def ExpandFileArgs(args):
335  """Replaces file-arg placeholders in args.
336
337  These placeholders have the form:
338    @FileArg(filename:key1:key2:...:keyn)
339
340  The value of such a placeholder is calculated by reading 'filename' as json.
341  And then extracting the value at [key1][key2]...[keyn].
342
343  Note: This intentionally does not return the list of files that appear in such
344  placeholders. An action that uses file-args *must* know the paths of those
345  files prior to the parsing of the arguments (typically by explicitly listing
346  them in the action's inputs in build files).
347  """
348  new_args = list(args)
349  file_jsons = dict()
350  r = re.compile('@FileArg\((.*?)\)')
351  for i, arg in enumerate(args):
352    match = r.search(arg)
353    if not match:
354      continue
355
356    if match.end() != len(arg):
357      raise Exception('Unexpected characters after FileArg: ' + arg)
358
359    lookup_path = match.group(1).split(':')
360    file_path = lookup_path[0]
361    if not file_path in file_jsons:
362      file_jsons[file_path] = ReadJson(file_path)
363
364    expansion = file_jsons[file_path]
365    for k in lookup_path[1:]:
366      expansion = expansion[k]
367
368    new_args[i] = arg[:match.start()] + str(expansion)
369
370  return new_args
371
372