• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The PDFium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Presubmit script for pdfium.
6
7See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
8for more details about the presubmit API built into depot_tools.
9"""
10
11PRESUBMIT_VERSION = '2.0.0'
12
13USE_PYTHON3 = True
14
15LINT_FILTERS = [
16  # Rvalue ref checks are unreliable.
17  '-build/c++11',
18  # Need to fix header names not matching cpp names.
19  '-build/include_order',
20  # Too many to fix at the moment.
21  '-readability/casting',
22  # Need to refactor large methods to fix.
23  '-readability/fn_size',
24  # Lots of usage to fix first.
25  '-runtime/int',
26  # Lots of non-const references need to be fixed
27  '-runtime/references',
28  # We are not thread safe, so this will never pass.
29  '-runtime/threadsafe_fn',
30  # Figure out how to deal with #defines that git cl format creates.
31  '-whitespace/indent',
32]
33
34
35_INCLUDE_ORDER_WARNING = (
36    'Your #include order seems to be broken. Remember to use the right '
37    'collation (LC_COLLATE=C) and check\nhttps://google.github.io/styleguide/'
38    'cppguide.html#Names_and_Order_of_Includes')
39
40
41# Bypass the AUTHORS check for these accounts.
42_KNOWN_ROBOTS = set() | set(
43    '%s@skia-public.iam.gserviceaccount.com' % s for s in ('pdfium-autoroll',))
44
45_THIRD_PARTY = 'third_party/'
46
47# Format: Sequence of tuples containing:
48# * String pattern or, if starting with a slash, a regular expression.
49# * Sequence of strings to show when the pattern matches.
50# * Error flag. True if a match is a presubmit error, otherwise it's a warning.
51# * Sequence of paths to *not* check (regexps).
52_BANNED_CPP_FUNCTIONS = (
53    (
54        r'/\busing namespace ',
55        (
56            'Using directives ("using namespace x") are banned by the Google',
57            'Style Guide (',
58            'https://google.github.io/styleguide/cppguide.html#Namespaces ).',
59            'Explicitly qualify symbols or use using declarations ("using',
60            'x::foo").',
61        ),
62        True,
63        [_THIRD_PARTY],
64    ),
65    (
66        r'/v8::Isolate::(?:|Try)GetCurrent\(\)',
67        (
68            'v8::Isolate::GetCurrent() and v8::Isolate::TryGetCurrent() are',
69            'banned. Hold a pointer to the v8::Isolate that was entered. Use',
70            'v8::Isolate::IsCurrent() to check whether a given v8::Isolate is',
71            'entered.',
72        ),
73        True,
74        (),
75    ),
76    (
77        r'/\bmemcpy\(',
78        ('Use FXSYS_memcpy() in place of memcpy().',),
79        True,
80        [_THIRD_PARTY],
81    ),
82    (
83        r'/\bmemmove\(',
84        ('Use FXSYS_memmove() in place of memmove().',),
85        True,
86        [_THIRD_PARTY],
87    ),
88    (
89        r'/\bmemset\(',
90        ('Use FXSYS_memset() in place of memset().',),
91        True,
92        [_THIRD_PARTY],
93    ),
94    (
95        r'/\bmemclr\(',
96        ('Use FXSYS_memclr() in place of memclr().',),
97        True,
98        [_THIRD_PARTY],
99    ),
100)
101
102
103def _CheckNoBannedFunctions(input_api, output_api):
104  """Makes sure that banned functions are not used."""
105  warnings = []
106  errors = []
107
108  def _GetMessageForMatchingType(input_api, affected_file, line_number, line,
109                                 type_name, message):
110    """Returns an string composed of the name of the file, the line number where
111    the match has been found and the additional text passed as `message` in case
112    the target type name matches the text inside the line passed as parameter.
113    """
114    result = []
115
116    if input_api.re.search(r"^ *//",
117                           line):  # Ignore comments about banned types.
118      return result
119    if line.endswith(
120        " nocheck"):  # A // nocheck comment will bypass this error.
121      return result
122
123    matched = False
124    if type_name[0:1] == '/':
125      regex = type_name[1:]
126      if input_api.re.search(regex, line):
127        matched = True
128    elif type_name in line:
129      matched = True
130
131    if matched:
132      result.append('    %s:%d:' % (affected_file.LocalPath(), line_number))
133      for message_line in message:
134        result.append('      %s' % message_line)
135
136    return result
137
138  def IsExcludedFile(affected_file, excluded_paths):
139    local_path = affected_file.LocalPath()
140    for item in excluded_paths:
141      if input_api.re.match(item, local_path):
142        return True
143    return False
144
145  def CheckForMatch(affected_file, line_num, line, func_name, message, error):
146    problems = _GetMessageForMatchingType(input_api, f, line_num, line,
147                                          func_name, message)
148    if problems:
149      if error:
150        errors.extend(problems)
151      else:
152        warnings.extend(problems)
153
154  file_filter = lambda f: f.LocalPath().endswith(('.cc', '.cpp', '.h'))
155  for f in input_api.AffectedFiles(file_filter=file_filter):
156    for line_num, line in f.ChangedContents():
157      for func_name, message, error, excluded_paths in _BANNED_CPP_FUNCTIONS:
158        if IsExcludedFile(f, excluded_paths):
159          continue
160        CheckForMatch(f, line_num, line, func_name, message, error)
161
162  result = []
163  if (warnings):
164    result.append(
165        output_api.PresubmitPromptWarning('Banned functions were used.\n' +
166                                          '\n'.join(warnings)))
167  if (errors):
168    result.append(
169        output_api.PresubmitError('Banned functions were used.\n' +
170                                  '\n'.join(errors)))
171  return result
172
173
174def _CheckUnwantedDependencies(input_api, output_api):
175  """Runs checkdeps on #include statements added in this
176  change. Breaking - rules is an error, breaking ! rules is a
177  warning.
178  """
179  import sys
180  # We need to wait until we have an input_api object and use this
181  # roundabout construct to import checkdeps because this file is
182  # eval-ed and thus doesn't have __file__.
183  original_sys_path = sys.path
184  try:
185    def GenerateCheckdepsPath(base_path):
186      return input_api.os_path.join(base_path, 'buildtools', 'checkdeps')
187
188    presubmit_path = input_api.PresubmitLocalPath()
189    presubmit_parent_path = input_api.os_path.dirname(presubmit_path)
190    not_standalone_pdfium = \
191        input_api.os_path.basename(presubmit_parent_path) == "third_party" and \
192        input_api.os_path.basename(presubmit_path) == "pdfium"
193
194    sys.path.append(GenerateCheckdepsPath(presubmit_path))
195    if not_standalone_pdfium:
196      presubmit_grandparent_path = input_api.os_path.dirname(
197          presubmit_parent_path)
198      sys.path.append(GenerateCheckdepsPath(presubmit_grandparent_path))
199
200    import checkdeps
201    from cpp_checker import CppChecker
202    from rules import Rule
203  except ImportError:
204    return [output_api.PresubmitError(
205        'Unable to run checkdeps, does pdfium/buildtools/checkdeps exist?')]
206  finally:
207    # Restore sys.path to what it was before.
208    sys.path = original_sys_path
209
210  added_includes = []
211  for f in input_api.AffectedFiles():
212    if not CppChecker.IsCppFile(f.LocalPath()):
213      continue
214
215    changed_lines = [line for line_num, line in f.ChangedContents()]
216    added_includes.append([f.LocalPath(), changed_lines])
217
218  deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath())
219
220  error_descriptions = []
221  warning_descriptions = []
222  for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes(
223      added_includes):
224    description_with_path = '%s\n    %s' % (path, rule_description)
225    if rule_type == Rule.DISALLOW:
226      error_descriptions.append(description_with_path)
227    else:
228      warning_descriptions.append(description_with_path)
229
230  results = []
231  if error_descriptions:
232    results.append(output_api.PresubmitError(
233        'You added one or more #includes that violate checkdeps rules.',
234        error_descriptions))
235  if warning_descriptions:
236    results.append(output_api.PresubmitPromptOrNotify(
237        'You added one or more #includes of files that are temporarily\n'
238        'allowed but being removed. Can you avoid introducing the\n'
239        '#include? See relevant DEPS file(s) for details and contacts.',
240        warning_descriptions))
241  return results
242
243
244def _CheckIncludeOrderForScope(scope, input_api, file_path, changed_linenums):
245  """Checks that the lines in scope occur in the right order.
246
247  1. C system files in alphabetical order
248  2. C++ system files in alphabetical order
249  3. Project's .h files
250  """
251
252  c_system_include_pattern = input_api.re.compile(r'\s*#include <.*\.h>')
253  cpp_system_include_pattern = input_api.re.compile(r'\s*#include <.*>')
254  custom_include_pattern = input_api.re.compile(r'\s*#include ".*')
255
256  C_SYSTEM_INCLUDES, CPP_SYSTEM_INCLUDES, CUSTOM_INCLUDES = range(3)
257
258  state = C_SYSTEM_INCLUDES
259
260  previous_line = ''
261  previous_line_num = 0
262  problem_linenums = []
263  out_of_order = " - line belongs before previous line"
264  for line_num, line in scope:
265    if c_system_include_pattern.match(line):
266      if state != C_SYSTEM_INCLUDES:
267        problem_linenums.append((line_num, previous_line_num,
268            " - C system include file in wrong block"))
269      elif previous_line and previous_line > line:
270        problem_linenums.append((line_num, previous_line_num,
271            out_of_order))
272    elif cpp_system_include_pattern.match(line):
273      if state == C_SYSTEM_INCLUDES:
274        state = CPP_SYSTEM_INCLUDES
275      elif state == CUSTOM_INCLUDES:
276        problem_linenums.append((line_num, previous_line_num,
277            " - c++ system include file in wrong block"))
278      elif previous_line and previous_line > line:
279        problem_linenums.append((line_num, previous_line_num, out_of_order))
280    elif custom_include_pattern.match(line):
281      if state != CUSTOM_INCLUDES:
282        state = CUSTOM_INCLUDES
283      elif previous_line and previous_line > line:
284        problem_linenums.append((line_num, previous_line_num, out_of_order))
285    else:
286      problem_linenums.append((line_num, previous_line_num,
287          "Unknown include type"))
288    previous_line = line
289    previous_line_num = line_num
290
291  warnings = []
292  for (line_num, previous_line_num, failure_type) in problem_linenums:
293    if line_num in changed_linenums or previous_line_num in changed_linenums:
294      warnings.append('    %s:%d:%s' % (file_path, line_num, failure_type))
295  return warnings
296
297
298def _CheckIncludeOrderInFile(input_api, f, changed_linenums):
299  """Checks the #include order for the given file f."""
300
301  system_include_pattern = input_api.re.compile(r'\s*#include \<.*')
302  # Exclude the following includes from the check:
303  # 1) #include <.../...>, e.g., <sys/...> includes often need to appear in a
304  # specific order.
305  # 2) <atlbase.h>, "build/build_config.h"
306  excluded_include_pattern = input_api.re.compile(
307      r'\s*#include (\<.*/.*|\<atlbase\.h\>|"build/build_config.h")')
308  custom_include_pattern = input_api.re.compile(r'\s*#include "(?P<FILE>.*)"')
309  # Match the final or penultimate token if it is xxxtest so we can ignore it
310  # when considering the special first include.
311  test_file_tag_pattern = input_api.re.compile(
312    r'_[a-z]+test(?=(_[a-zA-Z0-9]+)?\.)')
313  if_pattern = input_api.re.compile(
314      r'\s*#\s*(if|elif|else|endif|define|undef).*')
315  # Some files need specialized order of includes; exclude such files from this
316  # check.
317  uncheckable_includes_pattern = input_api.re.compile(
318      r'\s*#include '
319      '("ipc/.*macros\.h"|<windows\.h>|".*gl.*autogen.h")\s*')
320
321  contents = f.NewContents()
322  warnings = []
323  line_num = 0
324
325  # Handle the special first include. If the first include file is
326  # some/path/file.h, the corresponding including file can be some/path/file.cc,
327  # some/other/path/file.cc, some/path/file_platform.cc, some/path/file-suffix.h
328  # etc. It's also possible that no special first include exists.
329  # If the included file is some/path/file_platform.h the including file could
330  # also be some/path/file_xxxtest_platform.h.
331  including_file_base_name = test_file_tag_pattern.sub(
332    '', input_api.os_path.basename(f.LocalPath()))
333
334  for line in contents:
335    line_num += 1
336    if system_include_pattern.match(line):
337      # No special first include -> process the line again along with normal
338      # includes.
339      line_num -= 1
340      break
341    match = custom_include_pattern.match(line)
342    if match:
343      match_dict = match.groupdict()
344      header_basename = test_file_tag_pattern.sub(
345        '', input_api.os_path.basename(match_dict['FILE'])).replace('.h', '')
346
347      if header_basename not in including_file_base_name:
348        # No special first include -> process the line again along with normal
349        # includes.
350        line_num -= 1
351      break
352
353  # Split into scopes: Each region between #if and #endif is its own scope.
354  scopes = []
355  current_scope = []
356  for line in contents[line_num:]:
357    line_num += 1
358    if uncheckable_includes_pattern.match(line):
359      continue
360    if if_pattern.match(line):
361      scopes.append(current_scope)
362      current_scope = []
363    elif ((system_include_pattern.match(line) or
364           custom_include_pattern.match(line)) and
365          not excluded_include_pattern.match(line)):
366      current_scope.append((line_num, line))
367  scopes.append(current_scope)
368
369  for scope in scopes:
370    warnings.extend(_CheckIncludeOrderForScope(scope, input_api, f.LocalPath(),
371                                               changed_linenums))
372  return warnings
373
374
375def _CheckIncludeOrder(input_api, output_api):
376  """Checks that the #include order is correct.
377
378  1. The corresponding header for source files.
379  2. C system files in alphabetical order
380  3. C++ system files in alphabetical order
381  4. Project's .h files in alphabetical order
382
383  Each region separated by #if, #elif, #else, #endif, #define and #undef follows
384  these rules separately.
385  """
386  warnings = []
387  for f in input_api.AffectedFiles(file_filter=input_api.FilterSourceFile):
388    if f.LocalPath().endswith(('.cc', '.cpp', '.h', '.mm')):
389      changed_linenums = set(line_num for line_num, _ in f.ChangedContents())
390      warnings.extend(_CheckIncludeOrderInFile(input_api, f, changed_linenums))
391
392  results = []
393  if warnings:
394    results.append(output_api.PresubmitPromptOrNotify(_INCLUDE_ORDER_WARNING,
395                                                      warnings))
396  return results
397
398
399def _CheckLibcxxRevision(input_api, output_api):
400  """Makes sure that libcxx_revision is set correctly."""
401  if 'DEPS' not in [f.LocalPath() for f in input_api.AffectedFiles()]:
402    return []
403
404  script_path = input_api.os_path.join('testing', 'tools', 'libcxx_check.py')
405  buildtools_deps_path = input_api.os_path.join('buildtools',
406                                                'deps_revisions.gni')
407
408  try:
409    errors = input_api.subprocess.check_output(
410        [script_path, 'DEPS', buildtools_deps_path])
411  except input_api.subprocess.CalledProcessError as error:
412    msg = 'libcxx_check.py failed:'
413    long_text = error.output.decode('utf-8', 'ignore')
414    return [output_api.PresubmitError(msg, long_text=long_text)]
415
416  if errors:
417    return [output_api.PresubmitError(errors)]
418  return []
419
420
421def _CheckTestDuplicates(input_api, output_api):
422  """Checks that pixel and javascript tests don't contain duplicates.
423  We use .in and .pdf files, having both can cause race conditions on the bots,
424  which run the tests in parallel.
425  """
426  tests_added = []
427  results = []
428  for f in input_api.AffectedFiles():
429    if f.Action() == 'D':
430      continue
431    if not f.LocalPath().startswith(('testing/resources/pixel/',
432        'testing/resources/javascript/')):
433      continue
434    end_len = 0
435    if f.LocalPath().endswith('.in'):
436      end_len = 3
437    elif f.LocalPath().endswith('.pdf'):
438      end_len = 4
439    else:
440      continue
441    path = f.LocalPath()[:-end_len]
442    if path in tests_added:
443      results.append(output_api.PresubmitError(
444          'Remove %s to prevent shadowing %s' % (path + '.pdf',
445            path + '.in')))
446    else:
447      tests_added.append(path)
448  return results
449
450
451def _CheckPngNames(input_api, output_api):
452  """Checks that .png files have the right file name format, which must be in
453  the form:
454
455  NAME_expected(_gdi)?(_(agg|skia))?(_(linux|mac|win))?.pdf.\d+.png
456
457  This must be the same format as the one in testing/corpus's PRESUBMIT.py.
458  """
459  expected_pattern = input_api.re.compile(
460      r'.+_expected(_gdi)?(_(agg|skia))?(_(linux|mac|win))?\.pdf\.\d+.png')
461  results = []
462  for f in input_api.AffectedFiles(include_deletes=False):
463    if not f.LocalPath().endswith('.png'):
464      continue
465    if expected_pattern.match(f.LocalPath()):
466      continue
467    results.append(
468        output_api.PresubmitError(
469            'PNG file %s does not have the correct format' % f.LocalPath()))
470  return results
471
472
473def _CheckUselessForwardDeclarations(input_api, output_api):
474  """Checks that added or removed lines in non third party affected
475     header files do not lead to new useless class or struct forward
476     declaration.
477  """
478  results = []
479  class_pattern = input_api.re.compile(r'^class\s+(\w+);$',
480                                       input_api.re.MULTILINE)
481  struct_pattern = input_api.re.compile(r'^struct\s+(\w+);$',
482                                        input_api.re.MULTILINE)
483  for f in input_api.AffectedFiles(include_deletes=False):
484    if f.LocalPath().startswith('third_party'):
485      continue
486
487    if not f.LocalPath().endswith('.h'):
488      continue
489
490    contents = input_api.ReadFile(f)
491    fwd_decls = input_api.re.findall(class_pattern, contents)
492    fwd_decls.extend(input_api.re.findall(struct_pattern, contents))
493
494    useless_fwd_decls = []
495    for decl in fwd_decls:
496      count = sum(
497          1
498          for _ in input_api.re.finditer(r'\b%s\b' %
499                                         input_api.re.escape(decl), contents))
500      if count == 1:
501        useless_fwd_decls.append(decl)
502
503    if not useless_fwd_decls:
504      continue
505
506    for line in f.GenerateScmDiff().splitlines():
507      if (line.startswith('-') and not line.startswith('--') or
508          line.startswith('+') and not line.startswith('++')):
509        for decl in useless_fwd_decls:
510          if input_api.re.search(r'\b%s\b' % decl, line[1:]):
511            results.append(
512                output_api.PresubmitPromptWarning(
513                    '%s: %s forward declaration is no longer needed' %
514                    (f.LocalPath(), decl)))
515            useless_fwd_decls.remove(decl)
516
517  return results
518
519
520def ChecksCommon(input_api, output_api):
521  results = []
522
523  results.extend(
524      input_api.canned_checks.PanProjectChecks(
525          input_api, output_api, project_name='PDFium'))
526  results.extend(_CheckUnwantedDependencies(input_api, output_api))
527
528  # PanProjectChecks() doesn't consider .gn/.gni files, so check those, too.
529  files_to_check = (
530      r'.*\.gn$',
531      r'.*\.gni$',
532  )
533  results.extend(
534      input_api.canned_checks.CheckLicense(
535          input_api,
536          output_api,
537          project_name='PDFium',
538          source_file_filter=lambda x: input_api.FilterSourceFile(
539              x, files_to_check=files_to_check)))
540  results.extend(
541      input_api.canned_checks.CheckInclusiveLanguage(input_api, output_api))
542
543  return results
544
545
546def CheckChangeOnUpload(input_api, output_api):
547  results = []
548  results.extend(_CheckNoBannedFunctions(input_api, output_api))
549  results.extend(
550      input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
551  results.extend(
552      input_api.canned_checks.CheckChangeLintsClean(
553          input_api, output_api, lint_filters=LINT_FILTERS))
554  results.extend(_CheckIncludeOrder(input_api, output_api))
555  results.extend(_CheckLibcxxRevision(input_api, output_api))
556  results.extend(_CheckTestDuplicates(input_api, output_api))
557  results.extend(_CheckPngNames(input_api, output_api))
558  results.extend(_CheckUselessForwardDeclarations(input_api, output_api))
559
560  author = input_api.change.author_email
561  if author and author not in _KNOWN_ROBOTS:
562    results.extend(
563        input_api.canned_checks.CheckAuthorizedAuthor(input_api, output_api))
564
565  for f in input_api.AffectedFiles():
566    path, name = input_api.os_path.split(f.LocalPath())
567    if name == 'PRESUBMIT.py':
568      full_path = input_api.os_path.join(input_api.PresubmitLocalPath(), path)
569      test_file = input_api.os_path.join(path, 'PRESUBMIT_test.py')
570      if f.Action() != 'D' and input_api.os_path.exists(test_file):
571        # The PRESUBMIT.py file (and the directory containing it) might
572        # have been affected by being moved or removed, so only try to
573        # run the tests if they still exist.
574        results.extend(
575            input_api.canned_checks.RunUnitTestsInDirectory(
576                input_api,
577                output_api,
578                full_path,
579                files_to_check=[r'^PRESUBMIT_test\.py$'],
580                run_on_python2=not USE_PYTHON3,
581                run_on_python3=USE_PYTHON3,
582                skip_shebang_check=True))
583
584  return results
585