• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6"""Top-level presubmit script for Skia.
7
8See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
9for more details about the presubmit API built into gcl.
10"""
11
12import collections
13import csv
14import fnmatch
15import os
16import re
17import subprocess
18import sys
19import traceback
20
21
22REVERT_CL_SUBJECT_PREFIX = 'Revert '
23
24SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
25
26# Please add the complete email address here (and not just 'xyz@' or 'xyz').
27PUBLIC_API_OWNERS = (
28    'reed@chromium.org',
29    'reed@google.com',
30    'bsalomon@chromium.org',
31    'bsalomon@google.com',
32    'djsollen@chromium.org',
33    'djsollen@google.com',
34    'hcm@chromium.org',
35    'hcm@google.com',
36)
37
38AUTO_COMMIT_BOTS = (
39    'update-docs@skia.org',
40    'update-skps@skia.org'
41)
42
43AUTHORS_FILE_NAME = 'AUTHORS'
44
45DOCS_PREVIEW_URL = 'https://skia.org/?cl='
46GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
47
48# Path to CQ bots feature is described in https://bug.skia.org/4364
49PATH_PREFIX_TO_EXTRA_TRYBOTS = {
50    'src/opts/': ('skia.primary:'
51      'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SKNX_NO_SIMD'),
52    'include/private/SkAtomics.h': ('skia.primary:'
53      'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN,'
54      'Test-Ubuntu17-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-TSAN'
55    ),
56
57    # Below are examples to show what is possible with this feature.
58    # 'src/svg/': 'master1:abc;master2:def',
59    # 'src/svg/parser/': 'master3:ghi,jkl;master4:mno',
60    # 'src/image/SkImage_Base.h': 'master5:pqr,stu;master1:abc1;master2:def',
61}
62
63SERVICE_ACCOUNT_SUFFIX = '@skia-buildbots.google.com.iam.gserviceaccount.com'
64
65
66def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
67  """Checks that files end with atleast one \n (LF)."""
68  eof_files = []
69  for f in input_api.AffectedSourceFiles(source_file_filter):
70    contents = input_api.ReadFile(f, 'rb')
71    # Check that the file ends in atleast one newline character.
72    if len(contents) > 1 and contents[-1:] != '\n':
73      eof_files.append(f.LocalPath())
74
75  if eof_files:
76    return [output_api.PresubmitPromptWarning(
77      'These files should end in a newline character:',
78      items=eof_files)]
79  return []
80
81
82def _PythonChecks(input_api, output_api):
83  """Run checks on any modified Python files."""
84  pylint_disabled_files = (
85      'infra/bots/recipes.py',
86  )
87  pylint_disabled_warnings = (
88      'F0401',  # Unable to import.
89      'E0611',  # No name in module.
90      'W0232',  # Class has no __init__ method.
91      'E1002',  # Use of super on an old style class.
92      'W0403',  # Relative import used.
93      'R0201',  # Method could be a function.
94      'E1003',  # Using class name in super.
95      'W0613',  # Unused argument.
96      'W0105',  # String statement has no effect.
97  )
98  # Run Pylint on only the modified python files. Unfortunately it still runs
99  # Pylint on the whole file instead of just the modified lines.
100  affected_python_files = []
101  for affected_file in input_api.AffectedSourceFiles(None):
102    affected_file_path = affected_file.LocalPath()
103    if affected_file_path.endswith('.py'):
104      if affected_file_path not in pylint_disabled_files:
105        affected_python_files.append(affected_file_path)
106  return input_api.canned_checks.RunPylint(
107      input_api, output_api,
108      disabled_warnings=pylint_disabled_warnings,
109      white_list=affected_python_files)
110
111
112def _JsonChecks(input_api, output_api):
113  """Run checks on any modified json files."""
114  failing_files = []
115  for affected_file in input_api.AffectedFiles(None):
116    affected_file_path = affected_file.LocalPath()
117    is_json = affected_file_path.endswith('.json')
118    is_metadata = (affected_file_path.startswith('site/') and
119                   affected_file_path.endswith('/METADATA'))
120    if is_json or is_metadata:
121      try:
122        input_api.json.load(open(affected_file_path, 'r'))
123      except ValueError:
124        failing_files.append(affected_file_path)
125
126  results = []
127  if failing_files:
128    results.append(
129        output_api.PresubmitError(
130            'The following files contain invalid json:\n%s\n\n' %
131                '\n'.join(failing_files)))
132  return results
133
134
135def _IfDefChecks(input_api, output_api):
136  """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
137  comment_block_start_pattern = re.compile('^\s*\/\*.*$')
138  comment_block_middle_pattern = re.compile('^\s+\*.*')
139  comment_block_end_pattern = re.compile('^\s+\*\/.*$')
140  single_line_comment_pattern = re.compile('^\s*//.*$')
141  def is_comment(line):
142    return (comment_block_start_pattern.match(line) or
143            comment_block_middle_pattern.match(line) or
144            comment_block_end_pattern.match(line) or
145            single_line_comment_pattern.match(line))
146
147  empty_line_pattern = re.compile('^\s*$')
148  def is_empty_line(line):
149    return empty_line_pattern.match(line)
150
151  failing_files = []
152  for affected_file in input_api.AffectedSourceFiles(None):
153    affected_file_path = affected_file.LocalPath()
154    if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
155      f = open(affected_file_path)
156      for line in f.xreadlines():
157        if is_comment(line) or is_empty_line(line):
158          continue
159        # The below will be the first real line after comments and newlines.
160        if line.startswith('#if 0 '):
161          pass
162        elif line.startswith('#if ') or line.startswith('#ifdef '):
163          failing_files.append(affected_file_path)
164        break
165
166  results = []
167  if failing_files:
168    results.append(
169        output_api.PresubmitError(
170            'The following files have #if or #ifdef before includes:\n%s\n\n'
171            'See https://bug.skia.org/3362 for why this should be fixed.' %
172                '\n'.join(failing_files)))
173  return results
174
175
176def _CopyrightChecks(input_api, output_api, source_file_filter=None):
177  results = []
178  year_pattern = r'\d{4}'
179  year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
180  years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
181  copyright_pattern = (
182      r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
183
184  for affected_file in input_api.AffectedSourceFiles(source_file_filter):
185    if 'third_party' in affected_file.LocalPath():
186      continue
187    contents = input_api.ReadFile(affected_file, 'rb')
188    if not re.search(copyright_pattern, contents):
189      results.append(output_api.PresubmitError(
190          '%s is missing a correct copyright header.' % affected_file))
191  return results
192
193
194def _ToolFlags(input_api, output_api):
195  """Make sure `{dm,nanobench}_flags.py test` passes if modified."""
196  results = []
197  sources = lambda x: ('dm_flags.py'        in x.LocalPath() or
198                       'nanobench_flags.py' in x.LocalPath())
199  for f in input_api.AffectedSourceFiles(sources):
200    if 0 != subprocess.call(['python', f.LocalPath(), 'test']):
201      results.append(output_api.PresubmitError('`python %s test` failed' % f))
202  return results
203
204
205def _InfraTests(input_api, output_api):
206  """Run the infra tests."""
207  results = []
208  if not any(f.LocalPath().startswith('infra')
209             for f in input_api.AffectedFiles()):
210    return results
211
212  cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
213  try:
214    subprocess.check_output(cmd)
215  except subprocess.CalledProcessError as e:
216    results.append(output_api.PresubmitError(
217        '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
218  return results
219
220
221def _CheckGNFormatted(input_api, output_api):
222  """Make sure any .gn files we're changing have been formatted."""
223  results = []
224  for f in input_api.AffectedFiles():
225    if (not f.LocalPath().endswith('.gn') and
226        not f.LocalPath().endswith('.gni')):
227      continue
228
229    gn = 'gn.bat' if 'win32' in sys.platform else 'gn'
230    cmd = [gn, 'format', '--dry-run', f.LocalPath()]
231    try:
232      subprocess.check_output(cmd)
233    except subprocess.CalledProcessError:
234      fix = 'gn format ' + f.LocalPath()
235      results.append(output_api.PresubmitError(
236          '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
237  return results
238
239
240def _CommonChecks(input_api, output_api):
241  """Presubmit checks common to upload and commit."""
242  results = []
243  sources = lambda x: (x.LocalPath().endswith('.h') or
244                       x.LocalPath().endswith('.py') or
245                       x.LocalPath().endswith('.sh') or
246                       x.LocalPath().endswith('.m') or
247                       x.LocalPath().endswith('.mm') or
248                       x.LocalPath().endswith('.go') or
249                       x.LocalPath().endswith('.c') or
250                       x.LocalPath().endswith('.cc') or
251                       x.LocalPath().endswith('.cpp'))
252  results.extend(
253      _CheckChangeHasEol(
254          input_api, output_api, source_file_filter=sources))
255  results.extend(
256      input_api.canned_checks.CheckChangeHasNoCR(
257          input_api, output_api, source_file_filter=sources))
258  results.extend(
259      input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
260          input_api, output_api, source_file_filter=sources))
261  results.extend(_PythonChecks(input_api, output_api))
262  results.extend(_JsonChecks(input_api, output_api))
263  results.extend(_IfDefChecks(input_api, output_api))
264  results.extend(_CopyrightChecks(input_api, output_api,
265                                  source_file_filter=sources))
266  results.extend(_ToolFlags(input_api, output_api))
267  return results
268
269
270def CheckChangeOnUpload(input_api, output_api):
271  """Presubmit checks for the change on upload.
272
273  The following are the presubmit checks:
274  * Check change has one and only one EOL.
275  """
276  results = []
277  results.extend(_CommonChecks(input_api, output_api))
278  # Run on upload, not commit, since the presubmit bot apparently doesn't have
279  # coverage or Go installed.
280  results.extend(_InfraTests(input_api, output_api))
281
282  results.extend(_CheckGNFormatted(input_api, output_api))
283  return results
284
285
286def _CheckTreeStatus(input_api, output_api, json_url):
287  """Check whether to allow commit.
288
289  Args:
290    input_api: input related apis.
291    output_api: output related apis.
292    json_url: url to download json style status.
293  """
294  tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
295      input_api, output_api, json_url=json_url)
296  if not tree_status_results:
297    # Check for caution state only if tree is not closed.
298    connection = input_api.urllib2.urlopen(json_url)
299    status = input_api.json.loads(connection.read())
300    connection.close()
301    if ('caution' in status['message'].lower() and
302        os.isatty(sys.stdout.fileno())):
303      # Display a prompt only if we are in an interactive shell. Without this
304      # check the commit queue behaves incorrectly because it considers
305      # prompts to be failures.
306      short_text = 'Tree state is: ' + status['general_state']
307      long_text = status['message'] + '\n' + json_url
308      tree_status_results.append(
309          output_api.PresubmitPromptWarning(
310              message=short_text, long_text=long_text))
311  else:
312    # Tree status is closed. Put in message about contacting sheriff.
313    connection = input_api.urllib2.urlopen(
314        SKIA_TREE_STATUS_URL + '/current-sheriff')
315    sheriff_details = input_api.json.loads(connection.read())
316    if sheriff_details:
317      tree_status_results[0]._message += (
318          '\n\nPlease contact the current Skia sheriff (%s) if you are trying '
319          'to submit a build fix\nand do not know how to submit because the '
320          'tree is closed') % sheriff_details['username']
321  return tree_status_results
322
323
324class CodeReview(object):
325  """Abstracts which codereview tool is used for the specified issue."""
326
327  def __init__(self, input_api):
328    self._issue = input_api.change.issue
329    self._gerrit = input_api.gerrit
330
331  def GetOwnerEmail(self):
332    return self._gerrit.GetChangeOwner(self._issue)
333
334  def GetSubject(self):
335    return self._gerrit.GetChangeInfo(self._issue)['subject']
336
337  def GetDescription(self):
338    return self._gerrit.GetChangeDescription(self._issue)
339
340  def IsDryRun(self):
341    return self._gerrit.GetChangeInfo(
342        self._issue)['labels']['Commit-Queue'].get('value', 0) == 1
343
344  def GetReviewers(self):
345    code_review_label = (
346        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
347    return [r['email'] for r in code_review_label.get('all', [])]
348
349  def GetApprovers(self):
350    approvers = []
351    code_review_label = (
352        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
353    for m in code_review_label.get('all', []):
354      if m.get("value") == 1:
355        approvers.append(m["email"])
356    return approvers
357
358
359def _CheckOwnerIsInAuthorsFile(input_api, output_api):
360  results = []
361  if input_api.change.issue:
362    cr = CodeReview(input_api)
363
364    owner_email = cr.GetOwnerEmail()
365
366    # Service accounts don't need to be in AUTHORS.
367    if owner_email.endswith(SERVICE_ACCOUNT_SUFFIX):
368      return results
369
370    try:
371      authors_content = ''
372      for line in open(AUTHORS_FILE_NAME):
373        if not line.startswith('#'):
374          authors_content += line
375      email_fnmatches = re.findall('<(.*)>', authors_content)
376      for email_fnmatch in email_fnmatches:
377        if fnmatch.fnmatch(owner_email, email_fnmatch):
378          # Found a match, the user is in the AUTHORS file break out of the loop
379          break
380      else:
381        results.append(
382          output_api.PresubmitError(
383            'The email %s is not in Skia\'s AUTHORS file.\n'
384            'Issue owner, this CL must include an addition to the Skia AUTHORS '
385            'file.'
386            % owner_email))
387    except IOError:
388      # Do not fail if authors file cannot be found.
389      traceback.print_exc()
390      input_api.logging.error('AUTHORS file not found!')
391
392  return results
393
394
395def _CheckLGTMsForPublicAPI(input_api, output_api):
396  """Check LGTMs for public API changes.
397
398  For public API files make sure there is an LGTM from the list of owners in
399  PUBLIC_API_OWNERS.
400  """
401  results = []
402  requires_owner_check = False
403  for affected_file in input_api.AffectedFiles():
404    affected_file_path = affected_file.LocalPath()
405    file_path, file_ext = os.path.splitext(affected_file_path)
406    # We only care about files that end in .h and are under the top-level
407    # include dir, but not include/private.
408    if (file_ext == '.h' and
409        'include' == file_path.split(os.path.sep)[0] and
410        'private' not in file_path):
411      requires_owner_check = True
412
413  if not requires_owner_check:
414    return results
415
416  lgtm_from_owner = False
417  if input_api.change.issue:
418    cr = CodeReview(input_api)
419
420    if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
421      # It is a revert CL, ignore the public api owners check.
422      return results
423
424    if cr.IsDryRun():
425      # Ignore public api owners check for dry run CLs since they are not
426      # going to be committed.
427      return results
428
429    if input_api.gerrit:
430      for reviewer in cr.GetReviewers():
431        if reviewer in PUBLIC_API_OWNERS:
432          # If an owner is specified as an reviewer in Gerrit then ignore the
433          # public api owners check.
434          return results
435    else:
436      match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
437      if match:
438        tbr_section = match.group(1).strip().split(' ')[0]
439        tbr_entries = tbr_section.split(',')
440        for owner in PUBLIC_API_OWNERS:
441          if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
442            # If an owner is specified in the TBR= line then ignore the public
443            # api owners check.
444            return results
445
446    if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
447      # An owner created the CL that is an automatic LGTM.
448      lgtm_from_owner = True
449
450    for approver in cr.GetApprovers():
451      if approver in PUBLIC_API_OWNERS:
452        # Found an lgtm in a message from an owner.
453        lgtm_from_owner = True
454        break
455
456  if not lgtm_from_owner:
457    results.append(
458        output_api.PresubmitError(
459            "If this CL adds to or changes Skia's public API, you need an LGTM "
460            "from any of %s.  If this CL only removes from or doesn't change "
461            "Skia's public API, please add a short note to the CL saying so. "
462            "Add one of the owners as a reviewer to your CL as well as to the "
463            "TBR= line.  If you don't know if this CL affects Skia's public "
464            "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
465  return results
466
467
468def _FooterExists(footers, key, value):
469  for k, v in footers:
470    if k == key and v == value:
471      return True
472  return False
473
474
475def PostUploadHook(cl, change, output_api):
476  """git cl upload will call this hook after the issue is created/modified.
477
478  This hook does the following:
479  * Adds a link to preview docs changes if there are any docs changes in the CL.
480  * Adds 'No-Try: true' if the CL contains only docs changes.
481  * Adds 'No-Tree-Checks: true' for non master branch changes since they do not
482    need to be gated on the master branch's tree.
483  * Adds 'No-Try: true' for non master branch changes since trybots do not yet
484    work on them.
485  * Adds 'No-Presubmit: true' for non master branch changes since those don't
486    run the presubmit checks.
487  * Adds extra trybots for the paths defined in PATH_TO_EXTRA_TRYBOTS.
488  """
489
490  results = []
491  atleast_one_docs_change = False
492  all_docs_changes = True
493  for affected_file in change.AffectedFiles():
494    affected_file_path = affected_file.LocalPath()
495    file_path, _ = os.path.splitext(affected_file_path)
496    if 'site' == file_path.split(os.path.sep)[0]:
497      atleast_one_docs_change = True
498    else:
499      all_docs_changes = False
500    if atleast_one_docs_change and not all_docs_changes:
501      break
502
503  issue = cl.issue
504  if issue:
505    # Skip PostUploadHooks for all auto-commit bots. New patchsets (caused
506    # due to PostUploadHooks) invalidates the CQ+2 vote from the
507    # "--use-commit-queue" flag to "git cl upload".
508    if cl.GetIssueOwner() in AUTO_COMMIT_BOTS:
509      return results
510
511    original_description_lines, footers = cl.GetDescriptionFooters()
512    new_description_lines = list(original_description_lines)
513
514    # If the change includes only doc changes then add No-Try: true in the
515    # CL's description if it does not exist yet.
516    if all_docs_changes and not _FooterExists(footers, 'No-Try', 'true'):
517      new_description_lines.append('No-Try: true')
518      results.append(
519          output_api.PresubmitNotifyResult(
520              'This change has only doc changes. Automatically added '
521              '\'No-Try: true\' to the CL\'s description'))
522
523    # If there is atleast one docs change then add preview link in the CL's
524    # description if it does not already exist there.
525    docs_preview_link = '%s%s' % (DOCS_PREVIEW_URL, issue)
526    docs_preview_line = 'Docs-Preview: %s' % docs_preview_link
527    if (atleast_one_docs_change and
528        not _FooterExists(footers, 'Docs-Preview', docs_preview_link)):
529      # Automatically add a link to where the docs can be previewed.
530      new_description_lines.append(docs_preview_line)
531      results.append(
532          output_api.PresubmitNotifyResult(
533              'Automatically added a link to preview the docs changes to the '
534              'CL\'s description'))
535
536    # If the target ref is not master then add 'No-Tree-Checks: true' and
537    # 'No-Try: true' to the CL's description if it does not already exist there.
538    target_ref = cl.GetRemoteBranch()[1]
539    if target_ref != 'refs/remotes/origin/master':
540      if not _FooterExists(footers, 'No-Tree-Checks', 'true'):
541        new_description_lines.append('No-Tree-Checks: true')
542        results.append(
543            output_api.PresubmitNotifyResult(
544                'Branch changes do not need to rely on the master branch\'s '
545                'tree status. Automatically added \'No-Tree-Checks: true\' to '
546                'the CL\'s description'))
547      if not _FooterExists(footers, 'No-Try', 'true'):
548        new_description_lines.append('No-Try: true')
549        results.append(
550            output_api.PresubmitNotifyResult(
551                'Trybots do not yet work for non-master branches. '
552                'Automatically added \'No-Try: true\' to the CL\'s '
553                'description'))
554      if not _FooterExists(footers, 'No-Presubmit', 'true'):
555        new_description_lines.append('No-Presubmit: true')
556        results.append(
557            output_api.PresubmitNotifyResult(
558                'Branch changes do not run the presubmit checks.'))
559
560    # Automatically set Cq-Include-Trybots if any of the changed files here
561    # begin with the paths of interest.
562    bots_to_include = []
563    for affected_file in change.AffectedFiles():
564      affected_file_path = affected_file.LocalPath()
565      for path_prefix, extra_bots in PATH_PREFIX_TO_EXTRA_TRYBOTS.iteritems():
566        if affected_file_path.startswith(path_prefix):
567          results.append(
568              output_api.PresubmitNotifyResult(
569                  'Your CL modifies the path %s.\nAutomatically adding %s to '
570                  'the CL description.' % (affected_file_path, extra_bots)))
571          bots_to_include.append(extra_bots)
572    if bots_to_include:
573      output_api.EnsureCQIncludeTrybotsAreAdded(
574          cl, bots_to_include, new_description_lines)
575
576    # If the description has changed update it.
577    if new_description_lines != original_description_lines:
578      # Add a new line separating the new contents from the old contents.
579      new_description_lines.insert(len(original_description_lines), '')
580      cl.UpdateDescriptionFooters(new_description_lines, footers)
581
582    return results
583
584
585def CheckChangeOnCommit(input_api, output_api):
586  """Presubmit checks for the change on commit.
587
588  The following are the presubmit checks:
589  * Check change has one and only one EOL.
590  * Ensures that the Skia tree is open in
591    http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
592    state and an error if it is in 'Closed' state.
593  """
594  results = []
595  results.extend(_CommonChecks(input_api, output_api))
596  results.extend(
597      _CheckTreeStatus(input_api, output_api, json_url=(
598          SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
599  results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
600  results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
601  # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
602  # content of files.
603  results.extend(
604      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
605  return results
606