• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2019 Google LLC
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8
9import difflib
10import os
11import re
12import subprocess
13import sys
14
15
16# Any files in Git which match these patterns will be included, either directly
17# or indirectly via a parent dir.
18PATH_PATTERNS = [
19  r'.*\.c$',
20  r'.*\.cc$',
21  r'.*\.cpp$',
22  r'.*\.gn$',
23  r'.*\.gni$',
24  r'.*\.h$',
25]
26
27# These paths are always added to the inclusion list. Note that they may not
28# appear in the isolate if they are included indirectly via a parent dir.
29EXPLICIT_PATHS = [
30  '../.gclient',
31  '.clang-format',
32  '.clang-tidy',
33  'bin/fetch-clang-format',
34  'bin/fetch-gn',
35  'buildtools',
36  'infra/bots/assets/android_ndk_darwin/VERSION',
37  'infra/bots/assets/android_ndk_linux/VERSION',
38  'infra/bots/assets/android_ndk_windows/VERSION',
39  'infra/bots/assets/cast_toolchain/VERSION',
40  'infra/bots/assets/clang_linux/VERSION',
41  'infra/bots/assets/clang_win/VERSION',
42  'infra/bots/assets/mips64el_toolchain_linux/VERSION',
43  'infra/canvaskit',
44  'infra/pathkit',
45  'resources',
46  'third_party/externals',
47]
48
49# If a parent path contains more than this many immediate child paths (ie. files
50# and dirs which are directly inside it as opposed to indirect descendants), we
51# will include the parent in the isolate file instead of the children. This
52# results in a simpler isolate file which should need to be changed less often.
53COMBINE_PATHS_THRESHOLD = 3
54
55# Template for the isolate file content.
56ISOLATE_TMPL = '''{
57  'includes': [
58    'run_recipe.isolate',
59  ],
60  'variables': {
61    'files': [
62%s
63    ],
64  },
65}
66'''
67
68# Absolute path to the infra/bots dir.
69INFRABOTS_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
70
71# Absolute path to the compile.isolate file.
72ISOLATE_FILE = os.path.join(INFRABOTS_DIR, 'compile.isolate')
73
74
75def all_paths():
76  """Return all paths which are checked in to git."""
77  repo_root = os.path.abspath(os.path.join(INFRABOTS_DIR, os.pardir, os.pardir))
78  output = subprocess.check_output(['git', 'ls-files'], cwd=repo_root).rstrip()
79  return output.splitlines()
80
81
82def get_relevant_paths():
83  """Return all checked-in paths in PATH_PATTERNS or EXPLICIT_PATHS."""
84  paths = []
85  for f in all_paths():
86    for regexp in PATH_PATTERNS:
87      if re.match(regexp, f):
88        paths.append(f)
89        break
90
91  paths.extend(EXPLICIT_PATHS)
92  return paths
93
94
95class Tree(object):
96  """Tree helps with deduplicating and collapsing paths."""
97  class Node(object):
98    """Node represents an individual node in a Tree."""
99    def __init__(self, name):
100      self._children = {}
101      self._name = name
102      self._is_leaf = False
103
104    @property
105    def is_root(self):
106      """Return True iff this is the root node."""
107      return self._name is None
108
109    def add(self, entry):
110      """Add the given entry (given as a list of strings) to the Node."""
111      # Remove the first element if we're not the root node.
112      if not self.is_root:
113        if entry[0] != self._name:
114          raise ValueError('Cannot add a non-matching entry to a Node!')
115        entry = entry[1:]
116
117        # If the entry is now empty, this node is a leaf.
118        if not entry:
119          self._is_leaf = True
120          return
121
122      # Add a child node.
123      if not self._is_leaf:
124        child = self._children.get(entry[0])
125        if not child:
126          child = Tree.Node(entry[0])
127          self._children[entry[0]] = child
128        child.add(entry)
129
130        # If we have more than COMBINE_PATHS_THRESHOLD immediate children,
131        # combine them into this node.
132        immediate_children = 0
133        for child in self._children.itervalues():
134          if child._is_leaf:
135            immediate_children += 1
136        if not self.is_root and immediate_children >= COMBINE_PATHS_THRESHOLD:
137          self._is_leaf = True
138          self._children = {}
139
140    def entries(self):
141      """Return the entries represented by this node and its children.
142
143      Will not return children in the following cases:
144        - This Node is a leaf, ie. it represents an entry which was explicitly
145          inserted into the Tree, as opposed to only part of a path to other
146          entries.
147        - This Node has immediate children exceeding COMBINE_PATHS_THRESHOLD and
148          thus has been upgraded to a leaf node.
149      """
150      if self._is_leaf:
151        return [self._name]
152      rv = []
153      for child in self._children.itervalues():
154        for entry in child.entries():
155          if not self.is_root:
156            entry = self._name + '/' + entry
157          rv.append(entry)
158      return rv
159
160  def __init__(self):
161    self._root = Tree.Node(None)
162
163  def add(self, entry):
164    """Add the given entry to the tree."""
165    split = entry.split('/')
166    if split[-1] == '':
167      split = split[:-1]
168    self._root.add(split)
169
170  def entries(self):
171    """Return the list of entries in the tree.
172
173    Entries will be de-duplicated as follows:
174      - Any entry which is a sub-path of another entry will not be returned.
175      - Any entry which was not explicitly inserted but has children exceeding
176        the COMBINE_PATHS_THRESHOLD will be returned while its children will not
177        be returned.
178    """
179    return self._root.entries()
180
181
182def relpath(repo_path):
183  """Return a relative path to the given path within the repo.
184
185  The path is relative to the infra/bots dir, where the compile.isolate file
186  lives.
187  """
188  repo_path = '../../' + repo_path
189  repo_path = repo_path.replace('../../infra/', '../')
190  repo_path = repo_path.replace('../bots/', '')
191  return repo_path
192
193
194def get_isolate_content(paths):
195  """Construct the new content of the isolate file based on the given paths."""
196  lines = ['      \'%s\',' % relpath(p) for p in paths]
197  lines.sort()
198  return ISOLATE_TMPL % '\n'.join(lines)
199
200
201def main():
202  """Regenerate the compile.isolate file, or verify that it hasn't changed."""
203  testing = False
204  if len(sys.argv) == 2 and sys.argv[1] == 'test':
205    testing = True
206  elif len(sys.argv) != 1:
207    print >> sys.stderr, 'Usage: %s [test]' % sys.argv[0]
208    sys.exit(1)
209
210  tree = Tree()
211  for p in get_relevant_paths():
212    tree.add(p)
213  content = get_isolate_content(tree.entries())
214
215  if testing:
216    with open(ISOLATE_FILE, 'rb') as f:
217      expect_content = f.read()
218    if content != expect_content:
219      print >> sys.stderr, 'Found diff in %s:' % ISOLATE_FILE
220      a = expect_content.splitlines()
221      b = content.splitlines()
222      diff = difflib.context_diff(a, b, lineterm='')
223      for line in diff:
224        sys.stderr.write(line + '\n')
225      print >> sys.stderr, 'You may need to run:\n\n\tpython %s' % sys.argv[0]
226      sys.exit(1)
227  else:
228    with open(ISOLATE_FILE, 'wb') as f:
229      f.write(content)
230
231
232if __name__ == '__main__':
233  main()
234