• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30try:
31  import hashlib
32  md5er = hashlib.md5
33except ImportError, e:
34  import md5
35  md5er = md5.new
36
37
38import json
39import optparse
40import os
41from os.path import abspath, join, dirname, basename, exists
42import pickle
43import re
44import sys
45import subprocess
46import multiprocessing
47from subprocess import PIPE
48
49from testrunner.local import statusfile
50from testrunner.local import testsuite
51from testrunner.local import utils
52
53# Special LINT rules diverging from default and reason.
54# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
55# build/include_what_you_use: Started giving false positives for variables
56#   named "string" and "map" assuming that you needed to include STL headers.
57# TODO(bmeurer): Fix and re-enable readability/check
58# TODO(epertoso): Maybe re-enable readability/fn_size after
59# http://crrev.com/2199323003 relands.
60
61LINT_RULES = """
62-build/header_guard
63-build/include_what_you_use
64-build/namespaces
65-readability/check
66-readability/fn_size
67+readability/streams
68-runtime/references
69""".split()
70
71LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
72FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
73
74def CppLintWorker(command):
75  try:
76    process = subprocess.Popen(command, stderr=subprocess.PIPE)
77    process.wait()
78    out_lines = ""
79    error_count = -1
80    while True:
81      out_line = process.stderr.readline()
82      if out_line == '' and process.poll() != None:
83        if error_count == -1:
84          print "Failed to process %s" % command.pop()
85          return 1
86        break
87      m = LINT_OUTPUT_PATTERN.match(out_line)
88      if m:
89        out_lines += out_line
90        error_count += 1
91    sys.stdout.write(out_lines)
92    return error_count
93  except KeyboardInterrupt:
94    process.kill()
95  except:
96    print('Error running cpplint.py. Please make sure you have depot_tools' +
97          ' in your $PATH. Lint check skipped.')
98    process.kill()
99
100
101class FileContentsCache(object):
102
103  def __init__(self, sums_file_name):
104    self.sums = {}
105    self.sums_file_name = sums_file_name
106
107  def Load(self):
108    try:
109      sums_file = None
110      try:
111        sums_file = open(self.sums_file_name, 'r')
112        self.sums = pickle.load(sums_file)
113      except:
114        # Cannot parse pickle for any reason. Not much we can do about it.
115        pass
116    finally:
117      if sums_file:
118        sums_file.close()
119
120  def Save(self):
121    try:
122      sums_file = open(self.sums_file_name, 'w')
123      pickle.dump(self.sums, sums_file)
124    except:
125      # Failed to write pickle. Try to clean-up behind us.
126      if sums_file:
127        sums_file.close()
128      try:
129        os.unlink(self.sums_file_name)
130      except:
131        pass
132    finally:
133      sums_file.close()
134
135  def FilterUnchangedFiles(self, files):
136    changed_or_new = []
137    for file in files:
138      try:
139        handle = open(file, "r")
140        file_sum = md5er(handle.read()).digest()
141        if not file in self.sums or self.sums[file] != file_sum:
142          changed_or_new.append(file)
143          self.sums[file] = file_sum
144      finally:
145        handle.close()
146    return changed_or_new
147
148  def RemoveFile(self, file):
149    if file in self.sums:
150      self.sums.pop(file)
151
152
153class SourceFileProcessor(object):
154  """
155  Utility class that can run through a directory structure, find all relevant
156  files and invoke a custom check on the files.
157  """
158
159  def Run(self, path):
160    all_files = []
161    for file in self.GetPathsToSearch():
162      all_files += self.FindFilesIn(join(path, file))
163    if not self.ProcessFiles(all_files, path):
164      return False
165    return True
166
167  def IgnoreDir(self, name):
168    return (name.startswith('.') or
169            name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
170                     'octane', 'sunspider'))
171
172  def IgnoreFile(self, name):
173    return name.startswith('.')
174
175  def FindFilesIn(self, path):
176    result = []
177    for (root, dirs, files) in os.walk(path):
178      for ignored in [x for x in dirs if self.IgnoreDir(x)]:
179        dirs.remove(ignored)
180      for file in files:
181        if not self.IgnoreFile(file) and self.IsRelevant(file):
182          result.append(join(root, file))
183    return result
184
185
186class CppLintProcessor(SourceFileProcessor):
187  """
188  Lint files to check that they follow the google code style.
189  """
190
191  def IsRelevant(self, name):
192    return name.endswith('.cc') or name.endswith('.h')
193
194  def IgnoreDir(self, name):
195    return (super(CppLintProcessor, self).IgnoreDir(name)
196              or (name == 'third_party'))
197
198  IGNORE_LINT = ['flag-definitions.h']
199
200  def IgnoreFile(self, name):
201    return (super(CppLintProcessor, self).IgnoreFile(name)
202              or (name in CppLintProcessor.IGNORE_LINT))
203
204  def GetPathsToSearch(self):
205    return ['src', 'include', 'samples', join('test', 'cctest'),
206            join('test', 'unittests'), join('test', 'inspector')]
207
208  def GetCpplintScript(self, prio_path):
209    for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
210      path = path.strip('"')
211      cpplint = os.path.join(path, "cpplint.py")
212      if os.path.isfile(cpplint):
213        return cpplint
214
215    return None
216
217  def ProcessFiles(self, files, path):
218    good_files_cache = FileContentsCache('.cpplint-cache')
219    good_files_cache.Load()
220    files = good_files_cache.FilterUnchangedFiles(files)
221    if len(files) == 0:
222      print 'No changes in files detected. Skipping cpplint check.'
223      return True
224
225    filters = ",".join([n for n in LINT_RULES])
226    command = [sys.executable, 'cpplint.py', '--filter', filters]
227    cpplint = self.GetCpplintScript(join(path, "tools"))
228    if cpplint is None:
229      print('Could not find cpplint.py. Make sure '
230            'depot_tools is installed and in the path.')
231      sys.exit(1)
232
233    command = [sys.executable, cpplint, '--filter', filters]
234
235    commands = join([command + [file] for file in files])
236    count = multiprocessing.cpu_count()
237    pool = multiprocessing.Pool(count)
238    try:
239      results = pool.map_async(CppLintWorker, commands).get(999999)
240    except KeyboardInterrupt:
241      print "\nCaught KeyboardInterrupt, terminating workers."
242      sys.exit(1)
243
244    for i in range(len(files)):
245      if results[i] > 0:
246        good_files_cache.RemoveFile(files[i])
247
248    total_errors = sum(results)
249    print "Total errors found: %d" % total_errors
250    good_files_cache.Save()
251    return total_errors == 0
252
253
254COPYRIGHT_HEADER_PATTERN = re.compile(
255    r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
256
257class SourceProcessor(SourceFileProcessor):
258  """
259  Check that all files include a copyright notice and no trailing whitespaces.
260  """
261
262  RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c',
263                         '.status', '.gyp', '.gypi']
264
265  # Overwriting the one in the parent class.
266  def FindFilesIn(self, path):
267    if os.path.exists(path+'/.git'):
268      output = subprocess.Popen('git ls-files --full-name',
269                                stdout=PIPE, cwd=path, shell=True)
270      result = []
271      for file in output.stdout.read().split():
272        for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
273          if self.IgnoreDir(dir_part):
274            break
275        else:
276          if (self.IsRelevant(file) and os.path.exists(file)
277              and not self.IgnoreFile(file)):
278            result.append(join(path, file))
279      if output.wait() == 0:
280        return result
281    return super(SourceProcessor, self).FindFilesIn(path)
282
283  def IsRelevant(self, name):
284    for ext in SourceProcessor.RELEVANT_EXTENSIONS:
285      if name.endswith(ext):
286        return True
287    return False
288
289  def GetPathsToSearch(self):
290    return ['.']
291
292  def IgnoreDir(self, name):
293    return (super(SourceProcessor, self).IgnoreDir(name) or
294            name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources'))
295
296  IGNORE_COPYRIGHTS = ['box2d.js',
297                       'cpplint.py',
298                       'check_injected_script_source.py',
299                       'copy.js',
300                       'corrections.js',
301                       'crypto.js',
302                       'daemon.py',
303                       'debugger-script.js',
304                       'earley-boyer.js',
305                       'fannkuch.js',
306                       'fasta.js',
307                       'generate_protocol_externs.py',
308                       'injected-script.cc',
309                       'injected-script.h',
310                       'injected-script-source.js',
311                       'java-script-call-frame.cc',
312                       'java-script-call-frame.h',
313                       'jsmin.py',
314                       'libraries.cc',
315                       'libraries-empty.cc',
316                       'lua_binarytrees.js',
317                       'memops.js',
318                       'poppler.js',
319                       'primes.js',
320                       'raytrace.js',
321                       'regexp-pcre.js',
322                       'rjsmin.py',
323                       'script-breakpoint.h',
324                       'sqlite.js',
325                       'sqlite-change-heap.js',
326                       'sqlite-pointer-masking.js',
327                       'sqlite-safe-heap.js',
328                       'v8-debugger-script.h',
329                       'v8-function-call.cc',
330                       'v8-function-call.h',
331                       'v8-inspector-impl.cc',
332                       'v8-inspector-impl.h',
333                       'v8-runtime-agent-impl.cc',
334                       'v8-runtime-agent-impl.h',
335                       'gnuplot-4.6.3-emscripten.js',
336                       'zlib.js']
337  IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
338
339  def EndOfDeclaration(self, line):
340    return line == "}" or line == "};"
341
342  def StartOfDeclaration(self, line):
343    return line.find("//") == 0 or \
344           line.find("/*") == 0 or \
345           line.find(") {") != -1
346
347  def ProcessContents(self, name, contents):
348    result = True
349    base = basename(name)
350    if not base in SourceProcessor.IGNORE_TABS:
351      if '\t' in contents:
352        print "%s contains tabs" % name
353        result = False
354    if not base in SourceProcessor.IGNORE_COPYRIGHTS:
355      if not COPYRIGHT_HEADER_PATTERN.search(contents):
356        print "%s is missing a correct copyright header." % name
357        result = False
358    if ' \n' in contents or contents.endswith(' '):
359      line = 0
360      lines = []
361      parts = contents.split(' \n')
362      if not contents.endswith(' '):
363        parts.pop()
364      for part in parts:
365        line += part.count('\n') + 1
366        lines.append(str(line))
367      linenumbers = ', '.join(lines)
368      if len(lines) > 1:
369        print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
370      else:
371        print "%s has trailing whitespaces in line %s." % (name, linenumbers)
372      result = False
373    if not contents.endswith('\n') or contents.endswith('\n\n'):
374      print "%s does not end with a single new line." % name
375      result = False
376    # Sanitize flags for fuzzer.
377    if "mjsunit" in name:
378      match = FLAGS_LINE.search(contents)
379      if match:
380        print "%s Flags should use '-' (not '_')" % name
381        result = False
382    return result
383
384  def ProcessFiles(self, files, path):
385    success = True
386    violations = 0
387    for file in files:
388      try:
389        handle = open(file)
390        contents = handle.read()
391        if not self.ProcessContents(file, contents):
392          success = False
393          violations += 1
394      finally:
395        handle.close()
396    print "Total violating files: %s" % violations
397    return success
398
399def _CheckStatusFileForDuplicateKeys(filepath):
400  comma_space_bracket = re.compile(", *]")
401  lines = []
402  with open(filepath) as f:
403    for line in f.readlines():
404      # Skip all-comment lines.
405      if line.lstrip().startswith("#"): continue
406      # Strip away comments at the end of the line.
407      comment_start = line.find("#")
408      if comment_start != -1:
409        line = line[:comment_start]
410      line = line.strip()
411      # Strip away trailing commas within the line.
412      line = comma_space_bracket.sub("]", line)
413      if len(line) > 0:
414        lines.append(line)
415
416  # Strip away trailing commas at line ends. Ugh.
417  for i in range(len(lines) - 1):
418    if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
419        lines[i + 1][0] in ("}", "]")):
420      lines[i] = lines[i][:-1]
421
422  contents = "\n".join(lines)
423  # JSON wants double-quotes.
424  contents = contents.replace("'", '"')
425  # Fill in keywords (like PASS, SKIP).
426  for key in statusfile.KEYWORDS:
427    contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)
428
429  status = {"success": True}
430  def check_pairs(pairs):
431    keys = {}
432    for key, value in pairs:
433      if key in keys:
434        print("%s: Error: duplicate key %s" % (filepath, key))
435        status["success"] = False
436      keys[key] = True
437
438  json.loads(contents, object_pairs_hook=check_pairs)
439  return status["success"]
440
441def CheckStatusFiles(workspace):
442  success = True
443  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
444  for root in suite_paths:
445    suite_path = join(workspace, "test", root)
446    status_file_path = join(suite_path, root + ".status")
447    suite = testsuite.TestSuite.LoadTestSuite(suite_path)
448    if suite and exists(status_file_path):
449      success &= statusfile.PresubmitCheck(status_file_path)
450      success &= _CheckStatusFileForDuplicateKeys(status_file_path)
451  return success
452
453def CheckAuthorizedAuthor(input_api, output_api):
454  """For non-googler/chromites committers, verify the author's email address is
455  in AUTHORS.
456  """
457  # TODO(maruel): Add it to input_api?
458  import fnmatch
459
460  author = input_api.change.author_email
461  if not author:
462    input_api.logging.info('No author, skipping AUTHOR check')
463    return []
464  authors_path = input_api.os_path.join(
465      input_api.PresubmitLocalPath(), 'AUTHORS')
466  valid_authors = (
467      input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line)
468      for line in open(authors_path))
469  valid_authors = [item.group(1).lower() for item in valid_authors if item]
470  if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors):
471    input_api.logging.info('Valid authors are %s', ', '.join(valid_authors))
472    return [output_api.PresubmitPromptWarning(
473        ('%s is not in AUTHORS file. If you are a new contributor, please visit'
474        '\n'
475        'http://www.chromium.org/developers/contributing-code and read the '
476        '"Legal" section\n'
477        'If you are a chromite, verify the contributor signed the CLA.') %
478        author)]
479  return []
480
481def GetOptions():
482  result = optparse.OptionParser()
483  result.add_option('--no-lint', help="Do not run cpplint", default=False,
484                    action="store_true")
485  return result
486
487
488def Main():
489  workspace = abspath(join(dirname(sys.argv[0]), '..'))
490  parser = GetOptions()
491  (options, args) = parser.parse_args()
492  success = True
493  print "Running C++ lint check..."
494  if not options.no_lint:
495    success &= CppLintProcessor().Run(workspace)
496  print "Running copyright header, trailing whitespaces and " \
497        "two empty lines between declarations check..."
498  success &= SourceProcessor().Run(workspace)
499  success &= CheckStatusFiles(workspace)
500  if success:
501    return 0
502  else:
503    return 1
504
505
506if __name__ == '__main__':
507  sys.exit(Main())
508