• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2017 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Find header files missing in GN.
7
8This script gets all the header files from ninja_deps, which is from the true
9dependency generated by the compiler, and report if they don't exist in GN.
10"""
11
12import argparse
13import json
14import os
15import re
16import shutil
17import subprocess
18import sys
19import tempfile
20from multiprocessing import Process, Queue
21
22SRC_DIR = os.path.abspath(
23    os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
24DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
25
26SISO_PATH = os.path.join(SRC_DIR, 'third_party', 'siso', 'cipd', 'siso')
27NINJA_PATH = os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')
28
29
30def IsSisoUsed(out_dir):
31  return os.path.exists(os.path.join(out_dir, ".siso_deps"))
32
33
34def GetHeadersFromNinja(out_dir, skip_obj, q):
35  """Return all the header files from ninja_deps"""
36
37  def NinjaSource():
38    if IsSisoUsed(out_dir):
39      cmd = [
40          SISO_PATH,
41          'query',
42          'deps',
43          '-C',
44          out_dir,
45      ]
46    else:
47      cmd = [NINJA_PATH, '-C', out_dir, '-t', 'deps']
48    # A negative bufsize means to use the system default, which usually
49    # means fully buffered.
50    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
51    for line in iter(popen.stdout.readline, b''):
52      yield line.rstrip().decode('utf-8')
53
54    popen.stdout.close()
55    return_code = popen.wait()
56    if return_code:
57      raise subprocess.CalledProcessError(return_code, cmd)
58
59  ans, err = set(), None
60  try:
61    ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
62  except Exception as e:
63    err = str(e)
64  q.put((ans, err))
65
66
67def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
68  """Parse ninja output and get the header files"""
69  all_headers = {}
70
71  # Ninja always uses "/", even on Windows.
72  prefix = '../../'
73
74  is_valid = False
75  obj_file = ''
76  for line in ninja_out:
77    if line.startswith('    '):
78      if not is_valid:
79        continue
80      if line.endswith('.h') or line.endswith('.hh'):
81        f = line.strip()
82        if f.startswith(prefix):
83          f = f[6:]  # Remove the '../../' prefix
84          # build/ only contains build-specific files like build_config.h
85          # and buildflag.h, and system header files, so they should be
86          # skipped.
87          if f.startswith(out_dir) or f.startswith('out'):
88            continue
89          if not f.startswith('build'):
90            all_headers.setdefault(f, [])
91            if not skip_obj:
92              all_headers[f].append(obj_file)
93    else:
94      is_valid = line.endswith('(VALID)')
95      obj_file = line.split(':')[0]
96
97  return all_headers
98
99
100def GetHeadersFromGN(out_dir, q):
101  """Return all the header files from GN"""
102
103  tmp = None
104  ans, err = set(), None
105  try:
106    # Argument |dir| is needed to make sure it's on the same drive on Windows.
107    # dir='' means dir='.', but doesn't introduce an unneeded prefix.
108    tmp = tempfile.mkdtemp(dir='')
109    shutil.copy2(os.path.join(out_dir, 'args.gn'),
110                 os.path.join(tmp, 'args.gn'))
111    # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
112    gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
113    subprocess.check_call([
114        os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
115    gn_json = json.load(open(os.path.join(tmp, 'project.json')))
116    ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
117  except Exception as e:
118    err = str(e)
119  finally:
120    if tmp:
121      shutil.rmtree(tmp)
122  q.put((ans, err))
123
124
125def ParseGNProjectJSON(gn, out_dir, tmp_out):
126  """Parse GN output and get the header files"""
127  all_headers = set()
128
129  for _target, properties in gn['targets'].items():
130    sources = properties.get('sources', [])
131    public = properties.get('public', [])
132    # Exclude '"public": "*"'.
133    if type(public) is list:
134      sources += public
135    for f in sources:
136      if f.endswith('.h') or f.endswith('.hh'):
137        if f.startswith('//'):
138          f = f[2:]  # Strip the '//' prefix.
139          if f.startswith(tmp_out):
140            f = out_dir + f[len(tmp_out):]
141          all_headers.add(f)
142
143  return all_headers
144
145
146def GetDepsPrefixes(q):
147  """Return all the folders controlled by DEPS file"""
148  prefixes, err = set(), None
149  try:
150    gclient_exe = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
151    gclient_out = subprocess.check_output([
152        os.path.join(DEPOT_TOOLS_DIR,
153                     gclient_exe), 'recurse', '--no-progress', '-j1', 'python3',
154        '-c', 'import os;print(os.environ["GCLIENT_DEP_PATH"])'
155    ],
156                                          universal_newlines=True)
157    for i in gclient_out.split('\n'):
158      if i.startswith('src/'):
159        i = i[4:]
160        prefixes.add(i)
161  except Exception as e:
162    err = str(e)
163  q.put((prefixes, err))
164
165
166def IsBuildClean(out_dir):
167  if IsSisoUsed(out_dir):
168    cmd = [SISO_PATH, 'ninja', '-C', out_dir, '-n']
169  else:
170    cmd = [NINJA_PATH, '-C', out_dir, '-n']
171  try:
172    out = subprocess.check_output(cmd)
173    return b'no work to do.' in out
174  except Exception as e:
175    print(e)
176    return False
177
178
179def ParseAllowlist(allowlist):
180  out = set()
181  for line in allowlist.split('\n'):
182    line = re.sub(r'#.*', '', line).strip()
183    if line:
184      out.add(line)
185  return out
186
187
188def FilterOutDepsedRepo(files, deps):
189  return {f for f in files if not any(f.startswith(d) for d in deps)}
190
191
192def GetNonExistingFiles(lst):
193  out = set()
194  for f in lst:
195    if not os.path.isfile(f):
196      out.add(f)
197  return out
198
199
200def main():
201
202  def DumpJson(data):
203    if args.json:
204      with open(args.json, 'w') as f:
205        json.dump(data, f)
206
207  def PrintError(msg):
208    DumpJson([])
209    parser.error(msg)
210
211  parser = argparse.ArgumentParser(description='''
212      NOTE: Use ninja to build all targets in OUT_DIR before running
213      this script.''')
214  parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
215                      help='output directory of the build')
216  parser.add_argument('--json',
217                      help='JSON output filename for missing headers')
218  parser.add_argument('--allowlist', help='file containing allowlist')
219  parser.add_argument('--skip-dirty-check',
220                      action='store_true',
221                      help='skip checking whether the build is dirty')
222  parser.add_argument('--verbose', action='store_true',
223                      help='print more diagnostic info')
224
225  args, _extras = parser.parse_known_args()
226
227  if not os.path.isdir(args.out_dir):
228    parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
229
230  if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
231    dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
232    if args.json:
233      # Assume running on the bots. Silently skip this step.
234      # This is possible because "analyze" step can be wrong due to
235      # underspecified header files. See crbug.com/725877
236      print(dirty_msg)
237      DumpJson([])
238      return 0
239    else:
240      # Assume running interactively.
241      parser.error(dirty_msg)
242
243  d_q = Queue()
244  d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
245  d_p.start()
246
247  gn_q = Queue()
248  gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
249  gn_p.start()
250
251  deps_q = Queue()
252  deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
253  deps_p.start()
254
255  d, d_err = d_q.get()
256  gn, gn_err = gn_q.get()
257  missing = set(d.keys()) - gn
258  nonexisting = GetNonExistingFiles(gn)
259
260  deps, deps_err = deps_q.get()
261  missing = FilterOutDepsedRepo(missing, deps)
262  nonexisting = FilterOutDepsedRepo(nonexisting, deps)
263
264  d_p.join()
265  gn_p.join()
266  deps_p.join()
267
268  if d_err:
269    PrintError(d_err)
270  if gn_err:
271    PrintError(gn_err)
272  if deps_err:
273    PrintError(deps_err)
274  if len(GetNonExistingFiles(d)) > 0:
275    print('Non-existing files in ninja deps:', GetNonExistingFiles(d))
276    PrintError('Found non-existing files in ninja deps. You should ' +
277               'build all in OUT_DIR.')
278  if len(d) == 0:
279    PrintError('OUT_DIR looks empty. You should build all there.')
280  if any((('/gen/' in i) for i in nonexisting)):
281    PrintError('OUT_DIR looks wrong. You should build all there.')
282
283  if args.allowlist:
284    allowlist = ParseAllowlist(open(args.allowlist).read())
285    missing -= allowlist
286    nonexisting -= allowlist
287
288  missing = sorted(missing)
289  nonexisting = sorted(nonexisting)
290
291  DumpJson(sorted(missing + nonexisting))
292
293  if len(missing) == 0 and len(nonexisting) == 0:
294    return 0
295
296  if len(missing) > 0:
297    print('\nThe following files should be included in gn files:')
298    for i in missing:
299      print(i)
300
301  if len(nonexisting) > 0:
302    print('\nThe following non-existing files should be removed from gn files:')
303    for i in nonexisting:
304      print(i)
305
306  if args.verbose:
307    # Only get detailed obj dependency here since it is slower.
308    GetHeadersFromNinja(args.out_dir, False, d_q)
309    d, d_err = d_q.get()
310    print('\nDetailed dependency info:')
311    for f in missing:
312      print(f)
313      for cc in d[f]:
314        print('  ', cc)
315
316    print('\nMissing headers sorted by number of affected object files:')
317    count = {k: len(v) for (k, v) in d.items()}
318    for f in sorted(count, key=count.get, reverse=True):
319      if f in missing:
320        print(count[f], f)
321
322  if args.json:
323    # Assume running on the bots. Temporarily return 0 before
324    # https://crbug.com/937847 is fixed.
325    return 0
326  return 1
327
328
329if __name__ == '__main__':
330  sys.exit(main())
331