• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30try:
31  import hashlib
32  md5er = hashlib.md5
33except ImportError, e:
34  import md5
35  md5er = md5.new
36
37
38import json
39import optparse
40import os
41from os.path import abspath, join, dirname, basename, exists
42import pickle
43import re
44import sys
45import subprocess
46import multiprocessing
47from subprocess import PIPE
48
49from testrunner.local import statusfile
50from testrunner.local import testsuite
51from testrunner.local import utils
52
53# Special LINT rules diverging from default and reason.
54# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
55# build/include_what_you_use: Started giving false positives for variables
56#   named "string" and "map" assuming that you needed to include STL headers.
57# TODO(bmeurer): Fix and re-enable readability/check
58# http://crrev.com/2199323003 relands.
59
60LINT_RULES = """
61-build/header_guard
62-build/include_what_you_use
63-build/namespaces
64-readability/check
65-readability/fn_size
66+readability/streams
67-runtime/references
68""".split()
69
70LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
71FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
72ASSERT_OPTIMIZED_PATTERN = re.compile("assertOptimized")
73FLAGS_ENABLE_OPT = re.compile("//\s*Flags:.*--(crankshaft|turbo)[^-].*\n")
74ASSERT_UNOPTIMIZED_PATTERN = re.compile("assertUnoptimized")
75FLAGS_NO_ALWAYS_OPT = re.compile("//\s*Flags:.*--no-?always-opt.*\n")
76
77TOOLS_PATH = dirname(abspath(__file__))
78
79def CppLintWorker(command):
80  try:
81    process = subprocess.Popen(command, stderr=subprocess.PIPE)
82    process.wait()
83    out_lines = ""
84    error_count = -1
85    while True:
86      out_line = process.stderr.readline()
87      if out_line == '' and process.poll() != None:
88        if error_count == -1:
89          print "Failed to process %s" % command.pop()
90          return 1
91        break
92      m = LINT_OUTPUT_PATTERN.match(out_line)
93      if m:
94        out_lines += out_line
95        error_count += 1
96    sys.stdout.write(out_lines)
97    return error_count
98  except KeyboardInterrupt:
99    process.kill()
100  except:
101    print('Error running cpplint.py. Please make sure you have depot_tools' +
102          ' in your $PATH. Lint check skipped.')
103    process.kill()
104
105
106class FileContentsCache(object):
107
108  def __init__(self, sums_file_name):
109    self.sums = {}
110    self.sums_file_name = sums_file_name
111
112  def Load(self):
113    try:
114      sums_file = None
115      try:
116        sums_file = open(self.sums_file_name, 'r')
117        self.sums = pickle.load(sums_file)
118      except:
119        # Cannot parse pickle for any reason. Not much we can do about it.
120        pass
121    finally:
122      if sums_file:
123        sums_file.close()
124
125  def Save(self):
126    try:
127      sums_file = open(self.sums_file_name, 'w')
128      pickle.dump(self.sums, sums_file)
129    except:
130      # Failed to write pickle. Try to clean-up behind us.
131      if sums_file:
132        sums_file.close()
133      try:
134        os.unlink(self.sums_file_name)
135      except:
136        pass
137    finally:
138      sums_file.close()
139
140  def FilterUnchangedFiles(self, files):
141    changed_or_new = []
142    for file in files:
143      try:
144        handle = open(file, "r")
145        file_sum = md5er(handle.read()).digest()
146        if not file in self.sums or self.sums[file] != file_sum:
147          changed_or_new.append(file)
148          self.sums[file] = file_sum
149      finally:
150        handle.close()
151    return changed_or_new
152
153  def RemoveFile(self, file):
154    if file in self.sums:
155      self.sums.pop(file)
156
157
158class SourceFileProcessor(object):
159  """
160  Utility class that can run through a directory structure, find all relevant
161  files and invoke a custom check on the files.
162  """
163
164  def RunOnPath(self, path):
165    """Runs processor on all files under the given path."""
166
167    all_files = []
168    for file in self.GetPathsToSearch():
169      all_files += self.FindFilesIn(join(path, file))
170    return self.ProcessFiles(all_files)
171
172  def RunOnFiles(self, files):
173    """Runs processor only on affected files."""
174
175    # Helper for getting directory pieces.
176    dirs = lambda f: dirname(f).split(os.sep)
177
178    # Path offsets where to look (to be in sync with RunOnPath).
179    # Normalize '.' to check for it with str.startswith.
180    search_paths = [('' if p == '.' else p) for p in self.GetPathsToSearch()]
181
182    all_files = [
183      f.AbsoluteLocalPath()
184      for f in files
185      if (not self.IgnoreFile(f.LocalPath()) and
186          self.IsRelevant(f.LocalPath()) and
187          all(not self.IgnoreDir(d) for d in dirs(f.LocalPath())) and
188          any(map(f.LocalPath().startswith, search_paths)))
189    ]
190
191    return self.ProcessFiles(all_files)
192
193  def IgnoreDir(self, name):
194    return (name.startswith('.') or
195            name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
196                     'octane', 'sunspider'))
197
198  def IgnoreFile(self, name):
199    return name.startswith('.')
200
201  def FindFilesIn(self, path):
202    result = []
203    for (root, dirs, files) in os.walk(path):
204      for ignored in [x for x in dirs if self.IgnoreDir(x)]:
205        dirs.remove(ignored)
206      for file in files:
207        if not self.IgnoreFile(file) and self.IsRelevant(file):
208          result.append(join(root, file))
209    return result
210
211
212class CppLintProcessor(SourceFileProcessor):
213  """
214  Lint files to check that they follow the google code style.
215  """
216
217  def IsRelevant(self, name):
218    return name.endswith('.cc') or name.endswith('.h')
219
220  def IgnoreDir(self, name):
221    return (super(CppLintProcessor, self).IgnoreDir(name)
222              or (name == 'third_party'))
223
224  IGNORE_LINT = ['flag-definitions.h']
225
226  def IgnoreFile(self, name):
227    return (super(CppLintProcessor, self).IgnoreFile(name)
228              or (name in CppLintProcessor.IGNORE_LINT))
229
230  def GetPathsToSearch(self):
231    return ['src', 'include', 'samples', join('test', 'cctest'),
232            join('test', 'unittests'), join('test', 'inspector')]
233
234  def GetCpplintScript(self, prio_path):
235    for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
236      path = path.strip('"')
237      cpplint = os.path.join(path, "cpplint.py")
238      if os.path.isfile(cpplint):
239        return cpplint
240
241    return None
242
243  def ProcessFiles(self, files):
244    good_files_cache = FileContentsCache('.cpplint-cache')
245    good_files_cache.Load()
246    files = good_files_cache.FilterUnchangedFiles(files)
247    if len(files) == 0:
248      print 'No changes in files detected. Skipping cpplint check.'
249      return True
250
251    filters = ",".join([n for n in LINT_RULES])
252    command = [sys.executable, 'cpplint.py', '--filter', filters]
253    cpplint = self.GetCpplintScript(TOOLS_PATH)
254    if cpplint is None:
255      print('Could not find cpplint.py. Make sure '
256            'depot_tools is installed and in the path.')
257      sys.exit(1)
258
259    command = [sys.executable, cpplint, '--filter', filters]
260
261    commands = join([command + [file] for file in files])
262    count = multiprocessing.cpu_count()
263    pool = multiprocessing.Pool(count)
264    try:
265      results = pool.map_async(CppLintWorker, commands).get(999999)
266    except KeyboardInterrupt:
267      print "\nCaught KeyboardInterrupt, terminating workers."
268      sys.exit(1)
269
270    for i in range(len(files)):
271      if results[i] > 0:
272        good_files_cache.RemoveFile(files[i])
273
274    total_errors = sum(results)
275    print "Total errors found: %d" % total_errors
276    good_files_cache.Save()
277    return total_errors == 0
278
279
280COPYRIGHT_HEADER_PATTERN = re.compile(
281    r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
282
283class SourceProcessor(SourceFileProcessor):
284  """
285  Check that all files include a copyright notice and no trailing whitespaces.
286  """
287
288  RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c',
289                         '.status', '.gyp', '.gypi']
290
291  # Overwriting the one in the parent class.
292  def FindFilesIn(self, path):
293    if os.path.exists(path+'/.git'):
294      output = subprocess.Popen('git ls-files --full-name',
295                                stdout=PIPE, cwd=path, shell=True)
296      result = []
297      for file in output.stdout.read().split():
298        for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
299          if self.IgnoreDir(dir_part):
300            break
301        else:
302          if (self.IsRelevant(file) and os.path.exists(file)
303              and not self.IgnoreFile(file)):
304            result.append(join(path, file))
305      if output.wait() == 0:
306        return result
307    return super(SourceProcessor, self).FindFilesIn(path)
308
309  def IsRelevant(self, name):
310    for ext in SourceProcessor.RELEVANT_EXTENSIONS:
311      if name.endswith(ext):
312        return True
313    return False
314
315  def GetPathsToSearch(self):
316    return ['.']
317
318  def IgnoreDir(self, name):
319    return (super(SourceProcessor, self).IgnoreDir(name) or
320            name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources'))
321
322  IGNORE_COPYRIGHTS = ['box2d.js',
323                       'cpplint.py',
324                       'check_injected_script_source.py',
325                       'copy.js',
326                       'corrections.js',
327                       'crypto.js',
328                       'daemon.py',
329                       'debugger-script.js',
330                       'earley-boyer.js',
331                       'fannkuch.js',
332                       'fasta.js',
333                       'generate_protocol_externs.py',
334                       'injected-script.cc',
335                       'injected-script.h',
336                       'injected-script-source.js',
337                       'java-script-call-frame.cc',
338                       'java-script-call-frame.h',
339                       'jsmin.py',
340                       'libraries.cc',
341                       'libraries-empty.cc',
342                       'lua_binarytrees.js',
343                       'meta-123.js',
344                       'memops.js',
345                       'poppler.js',
346                       'primes.js',
347                       'raytrace.js',
348                       'regexp-pcre.js',
349                       'resources-123.js',
350                       'rjsmin.py',
351                       'script-breakpoint.h',
352                       'sqlite.js',
353                       'sqlite-change-heap.js',
354                       'sqlite-pointer-masking.js',
355                       'sqlite-safe-heap.js',
356                       'v8-debugger-script.h',
357                       'v8-function-call.cc',
358                       'v8-function-call.h',
359                       'v8-inspector-impl.cc',
360                       'v8-inspector-impl.h',
361                       'v8-runtime-agent-impl.cc',
362                       'v8-runtime-agent-impl.h',
363                       'gnuplot-4.6.3-emscripten.js',
364                       'zlib.js']
365  IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
366
367  IGNORE_COPYRIGHTS_DIRECTORY = "test/test262/local-tests"
368
369  def EndOfDeclaration(self, line):
370    return line == "}" or line == "};"
371
372  def StartOfDeclaration(self, line):
373    return line.find("//") == 0 or \
374           line.find("/*") == 0 or \
375           line.find(") {") != -1
376
377  def ProcessContents(self, name, contents):
378    result = True
379    base = basename(name)
380    if not base in SourceProcessor.IGNORE_TABS:
381      if '\t' in contents:
382        print "%s contains tabs" % name
383        result = False
384    if not base in SourceProcessor.IGNORE_COPYRIGHTS and \
385        not SourceProcessor.IGNORE_COPYRIGHTS_DIRECTORY in name:
386      if not COPYRIGHT_HEADER_PATTERN.search(contents):
387        print "%s is missing a correct copyright header." % name
388        result = False
389    if ' \n' in contents or contents.endswith(' '):
390      line = 0
391      lines = []
392      parts = contents.split(' \n')
393      if not contents.endswith(' '):
394        parts.pop()
395      for part in parts:
396        line += part.count('\n') + 1
397        lines.append(str(line))
398      linenumbers = ', '.join(lines)
399      if len(lines) > 1:
400        print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
401      else:
402        print "%s has trailing whitespaces in line %s." % (name, linenumbers)
403      result = False
404    if not contents.endswith('\n') or contents.endswith('\n\n'):
405      print "%s does not end with a single new line." % name
406      result = False
407    # Sanitize flags for fuzzer.
408    if "mjsunit" in name or "debugger" in name:
409      match = FLAGS_LINE.search(contents)
410      if match:
411        print "%s Flags should use '-' (not '_')" % name
412        result = False
413      if not "mjsunit/mjsunit.js" in name:
414        if ASSERT_OPTIMIZED_PATTERN.search(contents) and \
415            not FLAGS_ENABLE_OPT.search(contents):
416          print "%s Flag --crankshaft or --turbo should be set " \
417                "if assertOptimized() is used" % name
418          result = False
419        if ASSERT_UNOPTIMIZED_PATTERN.search(contents) and \
420            not FLAGS_NO_ALWAYS_OPT.search(contents):
421          print "%s Flag --no-always-opt should be set if " \
422                "assertUnoptimized() is used" % name
423          result = False
424    return result
425
426  def ProcessFiles(self, files):
427    success = True
428    violations = 0
429    for file in files:
430      try:
431        handle = open(file)
432        contents = handle.read()
433        if not self.ProcessContents(file, contents):
434          success = False
435          violations += 1
436      finally:
437        handle.close()
438    print "Total violating files: %s" % violations
439    return success
440
441def _CheckStatusFileForDuplicateKeys(filepath):
442  comma_space_bracket = re.compile(", *]")
443  lines = []
444  with open(filepath) as f:
445    for line in f.readlines():
446      # Skip all-comment lines.
447      if line.lstrip().startswith("#"): continue
448      # Strip away comments at the end of the line.
449      comment_start = line.find("#")
450      if comment_start != -1:
451        line = line[:comment_start]
452      line = line.strip()
453      # Strip away trailing commas within the line.
454      line = comma_space_bracket.sub("]", line)
455      if len(line) > 0:
456        lines.append(line)
457
458  # Strip away trailing commas at line ends. Ugh.
459  for i in range(len(lines) - 1):
460    if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
461        lines[i + 1][0] in ("}", "]")):
462      lines[i] = lines[i][:-1]
463
464  contents = "\n".join(lines)
465  # JSON wants double-quotes.
466  contents = contents.replace("'", '"')
467  # Fill in keywords (like PASS, SKIP).
468  for key in statusfile.KEYWORDS:
469    contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)
470
471  status = {"success": True}
472  def check_pairs(pairs):
473    keys = {}
474    for key, value in pairs:
475      if key in keys:
476        print("%s: Error: duplicate key %s" % (filepath, key))
477        status["success"] = False
478      keys[key] = True
479
480  json.loads(contents, object_pairs_hook=check_pairs)
481  return status["success"]
482
483
484class StatusFilesProcessor(SourceFileProcessor):
485  """Checks status files for incorrect syntax and duplicate keys."""
486
487  def IsRelevant(self, name):
488    # Several changes to files under the test directories could impact status
489    # files.
490    return True
491
492  def GetPathsToSearch(self):
493    return ['test']
494
495  def ProcessFiles(self, files):
496    test_path = join(dirname(TOOLS_PATH), 'test')
497    status_files = set([])
498    for file_path in files:
499      if file_path.startswith(test_path):
500        # Strip off absolute path prefix pointing to test suites.
501        pieces = file_path[len(test_path):].lstrip(os.sep).split(os.sep)
502        if pieces:
503          # Infer affected status file name. Only care for existing status
504          # files. Some directories under "test" don't have any.
505          if not os.path.isdir(join(test_path, pieces[0])):
506            continue
507          status_file = join(test_path, pieces[0], pieces[0] + ".status")
508          if not os.path.exists(status_file):
509            continue
510          status_files.add(status_file)
511
512    success = True
513    for status_file_path in sorted(status_files):
514      success &= statusfile.PresubmitCheck(status_file_path)
515      success &= _CheckStatusFileForDuplicateKeys(status_file_path)
516    return success
517
518
519def CheckDeps(workspace):
520  checkdeps_py = join(workspace, 'buildtools', 'checkdeps', 'checkdeps.py')
521  return subprocess.call([sys.executable, checkdeps_py, workspace]) == 0
522
523
524def GetOptions():
525  result = optparse.OptionParser()
526  result.add_option('--no-lint', help="Do not run cpplint", default=False,
527                    action="store_true")
528  return result
529
530
531def Main():
532  workspace = abspath(join(dirname(sys.argv[0]), '..'))
533  parser = GetOptions()
534  (options, args) = parser.parse_args()
535  success = True
536  print "Running checkdeps..."
537  success &= CheckDeps(workspace)
538  print "Running C++ lint check..."
539  if not options.no_lint:
540    success &= CppLintProcessor().RunOnPath(workspace)
541  print "Running copyright header, trailing whitespaces and " \
542        "two empty lines between declarations check..."
543  success &= SourceProcessor().RunOnPath(workspace)
544  print "Running status-files check..."
545  success &= StatusFilesProcessor().RunOnPath(workspace)
546  if success:
547    return 0
548  else:
549    return 1
550
551
552if __name__ == '__main__':
553  sys.exit(Main())
554