• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The PDFium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Presubmit script for pdfium.
6
7See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
8for more details about the presubmit API built into depot_tools.
9"""
10
11PRESUBMIT_VERSION = '2.0.0'
12
13USE_PYTHON3 = True
14
15LINT_FILTERS = [
16  # Rvalue ref checks are unreliable.
17  '-build/c++11',
18  # Need to fix header names not matching cpp names.
19  '-build/include_order',
20  # Too many to fix at the moment.
21  '-readability/casting',
22  # Need to refactor large methods to fix.
23  '-readability/fn_size',
24  # Lots of usage to fix first.
25  '-runtime/int',
26  # Lots of non-const references need to be fixed
27  '-runtime/references',
28  # We are not thread safe, so this will never pass.
29  '-runtime/threadsafe_fn',
30  # Figure out how to deal with #defines that git cl format creates.
31  '-whitespace/indent',
32]
33
34
35_INCLUDE_ORDER_WARNING = (
36    'Your #include order seems to be broken. Remember to use the right '
37    'collation (LC_COLLATE=C) and check\nhttps://google.github.io/styleguide/'
38    'cppguide.html#Names_and_Order_of_Includes')
39
40
41# Bypass the AUTHORS check for these accounts.
42_KNOWN_ROBOTS = set() | set(
43    '%s@skia-public.iam.gserviceaccount.com' % s for s in ('pdfium-autoroll',))
44
45_THIRD_PARTY = 'third_party/'
46
47# Format: Sequence of tuples containing:
48# * String pattern or, if starting with a slash, a regular expression.
49# * Sequence of strings to show when the pattern matches.
50# * Error flag. True if a match is a presubmit error, otherwise it's a warning.
51# * Sequence of paths to *not* check (regexps).
52_BANNED_CPP_FUNCTIONS = (
53    (
54        r'/\busing namespace ',
55        (
56            'Using directives ("using namespace x") are banned by the Google',
57            'Style Guide (',
58            'https://google.github.io/styleguide/cppguide.html#Namespaces ).',
59            'Explicitly qualify symbols or use using declarations ("using',
60            'x::foo").',
61        ),
62        True,
63        [_THIRD_PARTY],
64    ),
65    (
66        r'/v8::Isolate::(?:|Try)GetCurrent()',
67        (
68            'v8::Isolate::GetCurrent() and v8::Isolate::TryGetCurrent() are',
69            'banned. Hold a pointer to the v8::Isolate that was entered. Use',
70            'v8::Isolate::IsCurrent() to check whether a given v8::Isolate is',
71            'entered.',
72        ),
73        True,
74        (),
75    ),
76)
77
78
79def _CheckNoBannedFunctions(input_api, output_api):
80  """Makes sure that banned functions are not used."""
81  warnings = []
82  errors = []
83
84  def _GetMessageForMatchingType(input_api, affected_file, line_number, line,
85                                 type_name, message):
86    """Returns an string composed of the name of the file, the line number where
87    the match has been found and the additional text passed as `message` in case
88    the target type name matches the text inside the line passed as parameter.
89    """
90    result = []
91
92    if input_api.re.search(r"^ *//",
93                           line):  # Ignore comments about banned types.
94      return result
95    if line.endswith(
96        " nocheck"):  # A // nocheck comment will bypass this error.
97      return result
98
99    matched = False
100    if type_name[0:1] == '/':
101      regex = type_name[1:]
102      if input_api.re.search(regex, line):
103        matched = True
104    elif type_name in line:
105      matched = True
106
107    if matched:
108      result.append('    %s:%d:' % (affected_file.LocalPath(), line_number))
109      for message_line in message:
110        result.append('      %s' % message_line)
111
112    return result
113
114  def IsExcludedFile(affected_file, excluded_paths):
115    local_path = affected_file.LocalPath()
116    for item in excluded_paths:
117      if input_api.re.match(item, local_path):
118        return True
119    return False
120
121  def CheckForMatch(affected_file, line_num, line, func_name, message, error):
122    problems = _GetMessageForMatchingType(input_api, f, line_num, line,
123                                          func_name, message)
124    if problems:
125      if error:
126        errors.extend(problems)
127      else:
128        warnings.extend(problems)
129
130  file_filter = lambda f: f.LocalPath().endswith(('.cc', '.cpp', '.h'))
131  for f in input_api.AffectedFiles(file_filter=file_filter):
132    for line_num, line in f.ChangedContents():
133      for func_name, message, error, excluded_paths in _BANNED_CPP_FUNCTIONS:
134        if IsExcludedFile(f, excluded_paths):
135          continue
136        CheckForMatch(f, line_num, line, func_name, message, error)
137
138  result = []
139  if (warnings):
140    result.append(
141        output_api.PresubmitPromptWarning('Banned functions were used.\n' +
142                                          '\n'.join(warnings)))
143  if (errors):
144    result.append(
145        output_api.PresubmitError('Banned functions were used.\n' +
146                                  '\n'.join(errors)))
147  return result
148
149
150def _CheckUnwantedDependencies(input_api, output_api):
151  """Runs checkdeps on #include statements added in this
152  change. Breaking - rules is an error, breaking ! rules is a
153  warning.
154  """
155  import sys
156  # We need to wait until we have an input_api object and use this
157  # roundabout construct to import checkdeps because this file is
158  # eval-ed and thus doesn't have __file__.
159  original_sys_path = sys.path
160  try:
161    def GenerateCheckdepsPath(base_path):
162      return input_api.os_path.join(base_path, 'buildtools', 'checkdeps')
163
164    presubmit_path = input_api.PresubmitLocalPath()
165    presubmit_parent_path = input_api.os_path.dirname(presubmit_path)
166    not_standalone_pdfium = \
167        input_api.os_path.basename(presubmit_parent_path) == "third_party" and \
168        input_api.os_path.basename(presubmit_path) == "pdfium"
169
170    sys.path.append(GenerateCheckdepsPath(presubmit_path))
171    if not_standalone_pdfium:
172      presubmit_grandparent_path = input_api.os_path.dirname(
173          presubmit_parent_path)
174      sys.path.append(GenerateCheckdepsPath(presubmit_grandparent_path))
175
176    import checkdeps
177    from cpp_checker import CppChecker
178    from rules import Rule
179  except ImportError:
180    return [output_api.PresubmitError(
181        'Unable to run checkdeps, does pdfium/buildtools/checkdeps exist?')]
182  finally:
183    # Restore sys.path to what it was before.
184    sys.path = original_sys_path
185
186  added_includes = []
187  for f in input_api.AffectedFiles():
188    if not CppChecker.IsCppFile(f.LocalPath()):
189      continue
190
191    changed_lines = [line for line_num, line in f.ChangedContents()]
192    added_includes.append([f.LocalPath(), changed_lines])
193
194  deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath())
195
196  error_descriptions = []
197  warning_descriptions = []
198  for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes(
199      added_includes):
200    description_with_path = '%s\n    %s' % (path, rule_description)
201    if rule_type == Rule.DISALLOW:
202      error_descriptions.append(description_with_path)
203    else:
204      warning_descriptions.append(description_with_path)
205
206  results = []
207  if error_descriptions:
208    results.append(output_api.PresubmitError(
209        'You added one or more #includes that violate checkdeps rules.',
210        error_descriptions))
211  if warning_descriptions:
212    results.append(output_api.PresubmitPromptOrNotify(
213        'You added one or more #includes of files that are temporarily\n'
214        'allowed but being removed. Can you avoid introducing the\n'
215        '#include? See relevant DEPS file(s) for details and contacts.',
216        warning_descriptions))
217  return results
218
219
220def _CheckIncludeOrderForScope(scope, input_api, file_path, changed_linenums):
221  """Checks that the lines in scope occur in the right order.
222
223  1. C system files in alphabetical order
224  2. C++ system files in alphabetical order
225  3. Project's .h files
226  """
227
228  c_system_include_pattern = input_api.re.compile(r'\s*#include <.*\.h>')
229  cpp_system_include_pattern = input_api.re.compile(r'\s*#include <.*>')
230  custom_include_pattern = input_api.re.compile(r'\s*#include ".*')
231
232  C_SYSTEM_INCLUDES, CPP_SYSTEM_INCLUDES, CUSTOM_INCLUDES = range(3)
233
234  state = C_SYSTEM_INCLUDES
235
236  previous_line = ''
237  previous_line_num = 0
238  problem_linenums = []
239  out_of_order = " - line belongs before previous line"
240  for line_num, line in scope:
241    if c_system_include_pattern.match(line):
242      if state != C_SYSTEM_INCLUDES:
243        problem_linenums.append((line_num, previous_line_num,
244            " - C system include file in wrong block"))
245      elif previous_line and previous_line > line:
246        problem_linenums.append((line_num, previous_line_num,
247            out_of_order))
248    elif cpp_system_include_pattern.match(line):
249      if state == C_SYSTEM_INCLUDES:
250        state = CPP_SYSTEM_INCLUDES
251      elif state == CUSTOM_INCLUDES:
252        problem_linenums.append((line_num, previous_line_num,
253            " - c++ system include file in wrong block"))
254      elif previous_line and previous_line > line:
255        problem_linenums.append((line_num, previous_line_num, out_of_order))
256    elif custom_include_pattern.match(line):
257      if state != CUSTOM_INCLUDES:
258        state = CUSTOM_INCLUDES
259      elif previous_line and previous_line > line:
260        problem_linenums.append((line_num, previous_line_num, out_of_order))
261    else:
262      problem_linenums.append((line_num, previous_line_num,
263          "Unknown include type"))
264    previous_line = line
265    previous_line_num = line_num
266
267  warnings = []
268  for (line_num, previous_line_num, failure_type) in problem_linenums:
269    if line_num in changed_linenums or previous_line_num in changed_linenums:
270      warnings.append('    %s:%d:%s' % (file_path, line_num, failure_type))
271  return warnings
272
273
274def _CheckIncludeOrderInFile(input_api, f, changed_linenums):
275  """Checks the #include order for the given file f."""
276
277  system_include_pattern = input_api.re.compile(r'\s*#include \<.*')
278  # Exclude the following includes from the check:
279  # 1) #include <.../...>, e.g., <sys/...> includes often need to appear in a
280  # specific order.
281  # 2) <atlbase.h>, "build/build_config.h"
282  excluded_include_pattern = input_api.re.compile(
283      r'\s*#include (\<.*/.*|\<atlbase\.h\>|"build/build_config.h")')
284  custom_include_pattern = input_api.re.compile(r'\s*#include "(?P<FILE>.*)"')
285  # Match the final or penultimate token if it is xxxtest so we can ignore it
286  # when considering the special first include.
287  test_file_tag_pattern = input_api.re.compile(
288    r'_[a-z]+test(?=(_[a-zA-Z0-9]+)?\.)')
289  if_pattern = input_api.re.compile(
290      r'\s*#\s*(if|elif|else|endif|define|undef).*')
291  # Some files need specialized order of includes; exclude such files from this
292  # check.
293  uncheckable_includes_pattern = input_api.re.compile(
294      r'\s*#include '
295      '("ipc/.*macros\.h"|<windows\.h>|".*gl.*autogen.h")\s*')
296
297  contents = f.NewContents()
298  warnings = []
299  line_num = 0
300
301  # Handle the special first include. If the first include file is
302  # some/path/file.h, the corresponding including file can be some/path/file.cc,
303  # some/other/path/file.cc, some/path/file_platform.cc, some/path/file-suffix.h
304  # etc. It's also possible that no special first include exists.
305  # If the included file is some/path/file_platform.h the including file could
306  # also be some/path/file_xxxtest_platform.h.
307  including_file_base_name = test_file_tag_pattern.sub(
308    '', input_api.os_path.basename(f.LocalPath()))
309
310  for line in contents:
311    line_num += 1
312    if system_include_pattern.match(line):
313      # No special first include -> process the line again along with normal
314      # includes.
315      line_num -= 1
316      break
317    match = custom_include_pattern.match(line)
318    if match:
319      match_dict = match.groupdict()
320      header_basename = test_file_tag_pattern.sub(
321        '', input_api.os_path.basename(match_dict['FILE'])).replace('.h', '')
322
323      if header_basename not in including_file_base_name:
324        # No special first include -> process the line again along with normal
325        # includes.
326        line_num -= 1
327      break
328
329  # Split into scopes: Each region between #if and #endif is its own scope.
330  scopes = []
331  current_scope = []
332  for line in contents[line_num:]:
333    line_num += 1
334    if uncheckable_includes_pattern.match(line):
335      continue
336    if if_pattern.match(line):
337      scopes.append(current_scope)
338      current_scope = []
339    elif ((system_include_pattern.match(line) or
340           custom_include_pattern.match(line)) and
341          not excluded_include_pattern.match(line)):
342      current_scope.append((line_num, line))
343  scopes.append(current_scope)
344
345  for scope in scopes:
346    warnings.extend(_CheckIncludeOrderForScope(scope, input_api, f.LocalPath(),
347                                               changed_linenums))
348  return warnings
349
350
351def _CheckIncludeOrder(input_api, output_api):
352  """Checks that the #include order is correct.
353
354  1. The corresponding header for source files.
355  2. C system files in alphabetical order
356  3. C++ system files in alphabetical order
357  4. Project's .h files in alphabetical order
358
359  Each region separated by #if, #elif, #else, #endif, #define and #undef follows
360  these rules separately.
361  """
362  warnings = []
363  for f in input_api.AffectedFiles(file_filter=input_api.FilterSourceFile):
364    if f.LocalPath().endswith(('.cc', '.cpp', '.h', '.mm')):
365      changed_linenums = set(line_num for line_num, _ in f.ChangedContents())
366      warnings.extend(_CheckIncludeOrderInFile(input_api, f, changed_linenums))
367
368  results = []
369  if warnings:
370    results.append(output_api.PresubmitPromptOrNotify(_INCLUDE_ORDER_WARNING,
371                                                      warnings))
372  return results
373
374
375def _CheckLibcxxRevision(input_api, output_api):
376  """Makes sure that libcxx_revision is set correctly."""
377  if 'DEPS' not in [f.LocalPath() for f in input_api.AffectedFiles()]:
378    return []
379
380  script_path = input_api.os_path.join('testing', 'tools', 'libcxx_check.py')
381  buildtools_deps_path = input_api.os_path.join('buildtools',
382                                                'deps_revisions.gni')
383
384  try:
385    errors = input_api.subprocess.check_output(
386        [script_path, 'DEPS', buildtools_deps_path])
387  except input_api.subprocess.CalledProcessError as error:
388    msg = 'libcxx_check.py failed:'
389    long_text = error.output.decode('utf-8', 'ignore')
390    return [output_api.PresubmitError(msg, long_text=long_text)]
391
392  if errors:
393    return [output_api.PresubmitError(errors)]
394  return []
395
396
397def _CheckTestDuplicates(input_api, output_api):
398  """Checks that pixel and javascript tests don't contain duplicates.
399  We use .in and .pdf files, having both can cause race conditions on the bots,
400  which run the tests in parallel.
401  """
402  tests_added = []
403  results = []
404  for f in input_api.AffectedFiles():
405    if f.Action() == 'D':
406      continue
407    if not f.LocalPath().startswith(('testing/resources/pixel/',
408        'testing/resources/javascript/')):
409      continue
410    end_len = 0
411    if f.LocalPath().endswith('.in'):
412      end_len = 3
413    elif f.LocalPath().endswith('.pdf'):
414      end_len = 4
415    else:
416      continue
417    path = f.LocalPath()[:-end_len]
418    if path in tests_added:
419      results.append(output_api.PresubmitError(
420          'Remove %s to prevent shadowing %s' % (path + '.pdf',
421            path + '.in')))
422    else:
423      tests_added.append(path)
424  return results
425
426
427def _CheckPngNames(input_api, output_api):
428  """Checks that .png files have the right file name format, which must be in
429  the form:
430
431  NAME_expected(_(agg|skia))?(_(linux|mac|win))?.pdf.\d+.png
432
433  This must be the same format as the one in testing/corpus's PRESUBMIT.py.
434  """
435  expected_pattern = input_api.re.compile(
436      r'.+_expected(_(agg|skia))?(_(linux|mac|win))?\.pdf\.\d+.png')
437  results = []
438  for f in input_api.AffectedFiles(include_deletes=False):
439    if not f.LocalPath().endswith('.png'):
440      continue
441    if expected_pattern.match(f.LocalPath()):
442      continue
443    results.append(
444        output_api.PresubmitError(
445            'PNG file %s does not have the correct format' % f.LocalPath()))
446  return results
447
448
449def _CheckUselessForwardDeclarations(input_api, output_api):
450  """Checks that added or removed lines in non third party affected
451     header files do not lead to new useless class or struct forward
452     declaration.
453  """
454  results = []
455  class_pattern = input_api.re.compile(r'^class\s+(\w+);$',
456                                       input_api.re.MULTILINE)
457  struct_pattern = input_api.re.compile(r'^struct\s+(\w+);$',
458                                        input_api.re.MULTILINE)
459  for f in input_api.AffectedFiles(include_deletes=False):
460    if f.LocalPath().startswith('third_party'):
461      continue
462
463    if not f.LocalPath().endswith('.h'):
464      continue
465
466    contents = input_api.ReadFile(f)
467    fwd_decls = input_api.re.findall(class_pattern, contents)
468    fwd_decls.extend(input_api.re.findall(struct_pattern, contents))
469
470    useless_fwd_decls = []
471    for decl in fwd_decls:
472      count = sum(
473          1
474          for _ in input_api.re.finditer(r'\b%s\b' %
475                                         input_api.re.escape(decl), contents))
476      if count == 1:
477        useless_fwd_decls.append(decl)
478
479    if not useless_fwd_decls:
480      continue
481
482    for line in f.GenerateScmDiff().splitlines():
483      if (line.startswith('-') and not line.startswith('--') or
484          line.startswith('+') and not line.startswith('++')):
485        for decl in useless_fwd_decls:
486          if input_api.re.search(r'\b%s\b' % decl, line[1:]):
487            results.append(
488                output_api.PresubmitPromptWarning(
489                    '%s: %s forward declaration is no longer needed' %
490                    (f.LocalPath(), decl)))
491            useless_fwd_decls.remove(decl)
492
493  return results
494
495
496def ChecksCommon(input_api, output_api):
497  results = []
498
499  results.extend(
500      input_api.canned_checks.PanProjectChecks(
501          input_api, output_api, project_name='PDFium'))
502
503  # PanProjectChecks() doesn't consider .gn/.gni files, so check those, too.
504  files_to_check = (
505      r'.*\.gn$',
506      r'.*\.gni$',
507  )
508  results.extend(
509      input_api.canned_checks.CheckLicense(
510          input_api,
511          output_api,
512          project_name='PDFium',
513          source_file_filter=lambda x: input_api.FilterSourceFile(
514              x, files_to_check=files_to_check)))
515
516  return results
517
518
519def CheckChangeOnUpload(input_api, output_api):
520  results = []
521  results.extend(_CheckNoBannedFunctions(input_api, output_api))
522  results.extend(_CheckUnwantedDependencies(input_api, output_api))
523  results.extend(
524      input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
525  results.extend(
526      input_api.canned_checks.CheckChangeLintsClean(
527          input_api, output_api, lint_filters=LINT_FILTERS))
528  results.extend(_CheckIncludeOrder(input_api, output_api))
529  results.extend(_CheckLibcxxRevision(input_api, output_api))
530  results.extend(_CheckTestDuplicates(input_api, output_api))
531  results.extend(_CheckPngNames(input_api, output_api))
532  results.extend(_CheckUselessForwardDeclarations(input_api, output_api))
533
534  author = input_api.change.author_email
535  if author and author not in _KNOWN_ROBOTS:
536    results.extend(
537        input_api.canned_checks.CheckAuthorizedAuthor(input_api, output_api))
538
539  for f in input_api.AffectedFiles():
540    path, name = input_api.os_path.split(f.LocalPath())
541    if name == 'PRESUBMIT.py':
542      full_path = input_api.os_path.join(input_api.PresubmitLocalPath(), path)
543      test_file = input_api.os_path.join(path, 'PRESUBMIT_test.py')
544      if f.Action() != 'D' and input_api.os_path.exists(test_file):
545        # The PRESUBMIT.py file (and the directory containing it) might
546        # have been affected by being moved or removed, so only try to
547        # run the tests if they still exist.
548        results.extend(
549            input_api.canned_checks.RunUnitTestsInDirectory(
550                input_api,
551                output_api,
552                full_path,
553                files_to_check=[r'^PRESUBMIT_test\.py$'],
554                run_on_python2=not USE_PYTHON3,
555                run_on_python3=USE_PYTHON3,
556                skip_shebang_check=True))
557
558  return results
559