• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import glob
6import os
7import posixpath
8import re
9
10from pylib import constants
11
12_EXCLUSIONS = [
13    # Misc files that exist to document directories
14    re.compile(r'.*METADATA'),
15    re.compile(r'.*OWNERS'),
16    re.compile(r'.*\.md'),
17    re.compile(r'.*\.crx'),  # Chrome extension zip files.
18    re.compile(r'.*/\.git.*'),  # Any '.git*' directories/files.
19    re.compile(r'.*\.so'),  # Libraries packed into .apk.
20    re.compile(r'.*Mojo.*manifest\.json'),  # Some source_set()s pull these in.
21    re.compile(r'.*\.py'),  # Some test_support targets include python deps.
22    re.compile(r'.*\.apk'),  # Should be installed separately.
23    re.compile(r'.*\.jar'),  # Never need java intermediates.
24    re.compile(r'.*\.crx'),  # Used by download_from_google_storage.
25    re.compile(r'.*\.wpr'),  # Web-page-relay files needed only on host.
26    re.compile(r'.*lib.java/.*'),  # Never need java intermediates.
27
28    # Test filter files:
29    re.compile(r'.*/testing/buildbot/filters/.*'),
30
31    # Chrome external extensions config file.
32    re.compile(r'.*external_extensions\.json'),
33
34    # v8's blobs and icu data get packaged into APKs.
35    re.compile(r'.*snapshot_blob.*\.bin'),
36    re.compile(r'.*icudtl\.bin'),
37
38    # Scripts that are needed by swarming, but not on devices:
39    re.compile(r'.*llvm-symbolizer'),
40    re.compile(r'.*devil_util_(?:bin|dist)'),
41    re.compile(r'.*md5sum_(?:bin|dist)'),
42    re.compile(r'.*/development/scripts/stack'),
43    re.compile(r'.*/build/android/pylib/symbols'),
44    re.compile(r'.*/build/android/stacktrace'),
45
46    # Required for java deobfuscation on the host:
47    re.compile(r'.*build/android/stacktrace/.*'),
48    re.compile(r'.*third_party/jdk/.*'),
49    re.compile(r'.*third_party/proguard/.*'),
50
51    # Our tests don't need these.
52    re.compile(r'.*/devtools-frontend/.*front_end/.*'),
53
54    # Build artifacts:
55    re.compile(r'.*\.stamp'),
56    re.compile(r'.*\.pak\.info'),
57    re.compile(r'.*\.build_config.json'),
58    re.compile(r'.*\.incremental\.json'),
59]
60
61
62def _FilterDataDeps(abs_host_files):
63  exclusions = _EXCLUSIONS + [
64      re.compile(os.path.join(constants.GetOutDirectory(), 'bin'))
65  ]
66  return [p for p in abs_host_files if not any(r.match(p) for r in exclusions)]
67
68
69def DevicePathComponentsFor(host_path, output_directory=None):
70  """Returns the device path components for a given host path.
71
72  This returns the device path as a list of joinable path components,
73  with None as the first element to indicate that the path should be
74  rooted at $EXTERNAL_STORAGE.
75
76  e.g., given
77
78    '$RUNTIME_DEPS_ROOT_DIR/foo/bar/baz.txt'
79
80  this would return
81
82    [None, 'foo', 'bar', 'baz.txt']
83
84  This handles a couple classes of paths differently than it otherwise would:
85    - All .pak files get mapped to top-level paks/
86    - All other dependencies get mapped to the top level directory
87        - If a file is not in the output directory then it's relative path to
88          the output directory will start with .. strings, so we remove those
89          and then the path gets mapped to the top-level directory
90        - If a file is in the output directory then the relative path to the
91          output directory gets mapped to the top-level directory
92
93  e.g. given
94
95    '$RUNTIME_DEPS_ROOT_DIR/out/Release/icu_fake_dir/icudtl.dat'
96
97  this would return
98
99    [None, 'icu_fake_dir', 'icudtl.dat']
100
101  Args:
102    host_path: The absolute path to the host file.
103  Returns:
104    A list of device path components.
105  """
106  output_directory = output_directory or constants.GetOutDirectory()
107  if (host_path.startswith(output_directory) and
108      os.path.splitext(host_path)[1] == '.pak'):
109    return [None, 'paks', os.path.basename(host_path)]
110
111  rel_host_path = os.path.relpath(host_path, output_directory)
112
113  device_path_components = [None]
114  p = rel_host_path
115  while p:
116    p, d = os.path.split(p)
117    # The relative path from the output directory to a file under the runtime
118    # deps root directory may start with multiple .. strings, so they need to
119    # be skipped.
120    if d and d != os.pardir:
121      device_path_components.insert(1, d)
122  return device_path_components
123
124
125def GetDataDependencies(runtime_deps_path):
126  """Returns a list of device data dependencies.
127
128  Args:
129    runtime_deps_path: A str path to the .runtime_deps file.
130  Returns:
131    A list of (host_path, device_path) tuples.
132  """
133  if not runtime_deps_path:
134    return []
135
136  with open(runtime_deps_path, 'r') as runtime_deps_file:
137    # .runtime_deps can contain duplicates.
138    rel_host_files = sorted({l.strip() for l in runtime_deps_file if l})
139
140  output_directory = constants.GetOutDirectory()
141  abs_host_files = [
142      os.path.abspath(os.path.join(output_directory, r))
143      for r in rel_host_files]
144  filtered_abs_host_files = _FilterDataDeps(abs_host_files)
145  # TODO(crbug.com/40533647): Filter out host executables, and investigate
146  # whether other files could be filtered as well.
147  return [(f, DevicePathComponentsFor(f, output_directory))
148          for f in filtered_abs_host_files]
149
150
151def SubstituteDeviceRootSingle(device_path, device_root):
152  if not device_path:
153    return device_root
154  if isinstance(device_path, list):
155    return posixpath.join(*(p if p else device_root for p in device_path))
156  return device_path
157
158
159def SubstituteDeviceRoot(host_device_tuples, device_root):
160  return [(h, SubstituteDeviceRootSingle(d, device_root))
161          for h, d in host_device_tuples]
162
163
164def ExpandDataDependencies(host_device_tuples):
165  ret = []
166  for h, d in host_device_tuples:
167    if os.path.isdir(h):
168      for subpath in glob.glob(f'{h}/**', recursive=True):
169        if not os.path.isdir(subpath):
170          new_part = subpath[len(h):]
171          ret.append((subpath, d + new_part))
172    else:
173      ret.append((h, d))
174  return ret
175