• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6
7"""Top-level presubmit script for Skia.
8
9See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
10for more details about the presubmit API built into gcl.
11"""
12
13import fnmatch
14import os
15import re
16import subprocess
17import sys
18import traceback
19
20
21RELEASE_NOTES_FILE_NAME = 'RELEASE_NOTES.txt'
22
23GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
24
25SERVICE_ACCOUNT_SUFFIX = [
26    '@%s.iam.gserviceaccount.com' % project for project in [
27        'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public',
28        'skia-corp.google.com', 'chops-service-accounts']]
29
30
31def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
32  """Checks that files end with at least one \n (LF)."""
33  eof_files = []
34  for f in input_api.AffectedSourceFiles(source_file_filter):
35    contents = input_api.ReadFile(f, 'rb')
36    # Check that the file ends in at least one newline character.
37    if len(contents) > 1 and contents[-1:] != '\n':
38      eof_files.append(f.LocalPath())
39
40  if eof_files:
41    return [output_api.PresubmitPromptWarning(
42      'These files should end in a newline character:',
43      items=eof_files)]
44  return []
45
46
47def _JsonChecks(input_api, output_api):
48  """Run checks on any modified json files."""
49  failing_files = []
50  for affected_file in input_api.AffectedFiles(None):
51    affected_file_path = affected_file.LocalPath()
52    is_json = affected_file_path.endswith('.json')
53    is_metadata = (affected_file_path.startswith('site/') and
54                   affected_file_path.endswith('/METADATA'))
55    if is_json or is_metadata:
56      try:
57        input_api.json.load(open(affected_file_path, 'r'))
58      except ValueError:
59        failing_files.append(affected_file_path)
60
61  results = []
62  if failing_files:
63    results.append(
64        output_api.PresubmitError(
65            'The following files contain invalid json:\n%s\n\n' %
66                '\n'.join(failing_files)))
67  return results
68
69
70def _IfDefChecks(input_api, output_api):
71  """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
72  comment_block_start_pattern = re.compile('^\s*\/\*.*$')
73  comment_block_middle_pattern = re.compile('^\s+\*.*')
74  comment_block_end_pattern = re.compile('^\s+\*\/.*$')
75  single_line_comment_pattern = re.compile('^\s*//.*$')
76  def is_comment(line):
77    return (comment_block_start_pattern.match(line) or
78            comment_block_middle_pattern.match(line) or
79            comment_block_end_pattern.match(line) or
80            single_line_comment_pattern.match(line))
81
82  empty_line_pattern = re.compile('^\s*$')
83  def is_empty_line(line):
84    return empty_line_pattern.match(line)
85
86  failing_files = []
87  for affected_file in input_api.AffectedSourceFiles(None):
88    affected_file_path = affected_file.LocalPath()
89    if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
90      f = open(affected_file_path)
91      for line in f.xreadlines():
92        if is_comment(line) or is_empty_line(line):
93          continue
94        # The below will be the first real line after comments and newlines.
95        if line.startswith('#if 0 '):
96          pass
97        elif line.startswith('#if ') or line.startswith('#ifdef '):
98          failing_files.append(affected_file_path)
99        break
100
101  results = []
102  if failing_files:
103    results.append(
104        output_api.PresubmitError(
105            'The following files have #if or #ifdef before includes:\n%s\n\n'
106            'See https://bug.skia.org/3362 for why this should be fixed.' %
107                '\n'.join(failing_files)))
108  return results
109
110
111def _CopyrightChecks(input_api, output_api, source_file_filter=None):
112  results = []
113  year_pattern = r'\d{4}'
114  year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
115  years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
116  copyright_pattern = (
117      r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
118
119  for affected_file in input_api.AffectedSourceFiles(source_file_filter):
120    if ('third_party/' in affected_file.LocalPath() or
121        'tests/sksl/' in affected_file.LocalPath()):
122      continue
123    contents = input_api.ReadFile(affected_file, 'rb')
124    if not re.search(copyright_pattern, contents):
125      results.append(output_api.PresubmitError(
126          '%s is missing a correct copyright header.' % affected_file))
127  return results
128
129
130def _InfraTests(input_api, output_api):
131  """Run the infra tests."""
132  results = []
133  if not any(f.LocalPath().startswith('infra')
134             for f in input_api.AffectedFiles()):
135    return results
136
137  cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
138  try:
139    subprocess.check_output(cmd)
140  except subprocess.CalledProcessError as e:
141    results.append(output_api.PresubmitError(
142        '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
143  return results
144
145
146def _CheckGNFormatted(input_api, output_api):
147  """Make sure any .gn files we're changing have been formatted."""
148  files = []
149  for f in input_api.AffectedFiles(include_deletes=False):
150    if (f.LocalPath().endswith('.gn') or
151        f.LocalPath().endswith('.gni')):
152      files.append(f)
153  if not files:
154    return []
155
156  cmd = ['python', os.path.join('bin', 'fetch-gn')]
157  try:
158    subprocess.check_output(cmd)
159  except subprocess.CalledProcessError as e:
160    return [output_api.PresubmitError(
161        '`%s` failed:\n%s' % (' '.join(cmd), e.output))]
162
163  results = []
164  for f in files:
165    gn = 'gn.exe' if 'win32' in sys.platform else 'gn'
166    gn = os.path.join(input_api.PresubmitLocalPath(), 'bin', gn)
167    cmd = [gn, 'format', '--dry-run', f.LocalPath()]
168    try:
169      subprocess.check_output(cmd)
170    except subprocess.CalledProcessError:
171      fix = 'bin/gn format ' + f.LocalPath()
172      results.append(output_api.PresubmitError(
173          '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
174  return results
175
176
177def _CheckGitConflictMarkers(input_api, output_api):
178  pattern = input_api.re.compile('^(?:<<<<<<<|>>>>>>>) |^=======$')
179  results = []
180  for f in input_api.AffectedFiles():
181    for line_num, line in f.ChangedContents():
182      if f.LocalPath().endswith('.md'):
183        # First-level headers in markdown look a lot like version control
184        # conflict markers. http://daringfireball.net/projects/markdown/basics
185        continue
186      if pattern.match(line):
187        results.append(
188            output_api.PresubmitError(
189                'Git conflict markers found in %s:%d %s' % (
190                    f.LocalPath(), line_num, line)))
191  return results
192
193
194def _CheckIncludesFormatted(input_api, output_api):
195  """Make sure #includes in files we're changing have been formatted."""
196  files = [str(f) for f in input_api.AffectedFiles() if f.Action() != 'D']
197  cmd = ['python',
198         'tools/rewrite_includes.py',
199         '--dry-run'] + files
200  if 0 != subprocess.call(cmd):
201    return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))]
202  return []
203
204
205class _WarningsAsErrors():
206  def __init__(self, output_api):
207    self.output_api = output_api
208    self.old_warning = None
209  def __enter__(self):
210    self.old_warning = self.output_api.PresubmitPromptWarning
211    self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError
212    return self.output_api
213  def __exit__(self, ex_type, ex_value, ex_traceback):
214    self.output_api.PresubmitPromptWarning = self.old_warning
215
216
217def _CheckDEPSValid(input_api, output_api):
218  """Ensure that DEPS contains valid entries."""
219  results = []
220  script = os.path.join('infra', 'bots', 'check_deps.py')
221  relevant_files = ('DEPS', script)
222  for f in input_api.AffectedFiles():
223    if f.LocalPath() in relevant_files:
224      break
225  else:
226    return results
227  cmd = ['python', script]
228  try:
229    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
230  except subprocess.CalledProcessError as e:
231    results.append(output_api.PresubmitError(e.output))
232  return results
233
234
235def _RegenerateAllExamplesCPP(input_api, output_api):
236  """Regenerates all_examples.cpp if an example was added or deleted."""
237  if not any(f.LocalPath().startswith('docs/examples/')
238             for f in input_api.AffectedFiles()):
239    return []
240  command_str = 'tools/fiddle/make_all_examples_cpp.py'
241  cmd = ['python', command_str]
242  if 0 != subprocess.call(cmd):
243    return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))]
244
245  results = []
246  git_diff_output = input_api.subprocess.check_output(
247      ['git', 'diff', '--no-ext-diff'])
248  if git_diff_output:
249    results += [output_api.PresubmitError(
250        'Diffs found after running "%s":\n\n%s\n'
251        'Please commit or discard the above changes.' % (
252            command_str,
253            git_diff_output,
254        )
255    )]
256  return results
257
258def _CommonChecks(input_api, output_api):
259  """Presubmit checks common to upload and commit."""
260  results = []
261  sources = lambda x: (x.LocalPath().endswith('.h') or
262                       x.LocalPath().endswith('.py') or
263                       x.LocalPath().endswith('.sh') or
264                       x.LocalPath().endswith('.m') or
265                       x.LocalPath().endswith('.mm') or
266                       x.LocalPath().endswith('.go') or
267                       x.LocalPath().endswith('.c') or
268                       x.LocalPath().endswith('.cc') or
269                       x.LocalPath().endswith('.cpp'))
270  results.extend(_CheckChangeHasEol(
271      input_api, output_api, source_file_filter=sources))
272  with _WarningsAsErrors(output_api):
273    results.extend(input_api.canned_checks.CheckChangeHasNoCR(
274        input_api, output_api, source_file_filter=sources))
275    results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
276        input_api, output_api, source_file_filter=sources))
277  results.extend(_JsonChecks(input_api, output_api))
278  results.extend(_IfDefChecks(input_api, output_api))
279  results.extend(_CopyrightChecks(input_api, output_api,
280                                  source_file_filter=sources))
281  results.extend(_CheckDEPSValid(input_api, output_api))
282  results.extend(_CheckIncludesFormatted(input_api, output_api))
283  results.extend(_CheckGNFormatted(input_api, output_api))
284  results.extend(_CheckGitConflictMarkers(input_api, output_api))
285  results.extend(_RegenerateAllExamplesCPP(input_api, output_api))
286  return results
287
288
289def CheckChangeOnUpload(input_api, output_api):
290  """Presubmit checks for the change on upload."""
291  results = []
292  results.extend(_CommonChecks(input_api, output_api))
293  # Run on upload, not commit, since the presubmit bot apparently doesn't have
294  # coverage or Go installed.
295  results.extend(_InfraTests(input_api, output_api))
296  results.extend(_CheckReleaseNotesForPublicAPI(input_api, output_api))
297  return results
298
299
300class CodeReview(object):
301  """Abstracts which codereview tool is used for the specified issue."""
302
303  def __init__(self, input_api):
304    self._issue = input_api.change.issue
305    self._gerrit = input_api.gerrit
306
307  def GetOwnerEmail(self):
308    return self._gerrit.GetChangeOwner(self._issue)
309
310  def GetSubject(self):
311    return self._gerrit.GetChangeInfo(self._issue)['subject']
312
313  def GetDescription(self):
314    return self._gerrit.GetChangeDescription(self._issue)
315
316  def GetReviewers(self):
317    code_review_label = (
318        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
319    return [r['email'] for r in code_review_label.get('all', [])]
320
321  def GetApprovers(self):
322    approvers = []
323    code_review_label = (
324        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
325    for m in code_review_label.get('all', []):
326      if m.get("value") == 1:
327        approvers.append(m["email"])
328    return approvers
329
330
331def _CheckReleaseNotesForPublicAPI(input_api, output_api):
332  """Checks to see if release notes file is updated with public API changes."""
333  results = []
334  public_api_changed = False
335  release_file_changed = False
336  for affected_file in input_api.AffectedFiles():
337    affected_file_path = affected_file.LocalPath()
338    file_path, file_ext = os.path.splitext(affected_file_path)
339    # We only care about files that end in .h and are under the top-level
340    # include dir, but not include/private.
341    if (file_ext == '.h' and
342        file_path.split(os.path.sep)[0] == 'include' and
343        'private' not in file_path):
344      public_api_changed = True
345    elif affected_file_path == RELEASE_NOTES_FILE_NAME:
346      release_file_changed = True
347
348  if public_api_changed and not release_file_changed:
349    results.append(output_api.PresubmitPromptWarning(
350        'If this change affects a client API, please add a summary line '
351        'to the %s file.' % RELEASE_NOTES_FILE_NAME))
352  return results
353
354
355def PostUploadHook(gerrit, change, output_api):
356  """git cl upload will call this hook after the issue is created/modified.
357
358  This hook does the following:
359  * Adds a link to preview docs changes if there are any docs changes in the CL.
360  * Adds 'No-Try: true' if the CL contains only docs changes.
361  """
362  if not change.issue:
363    return []
364
365  # Skip PostUploadHooks for all auto-commit service account bots. New
366  # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
367  # the "--use-commit-queue" flag to "git cl upload".
368  for suffix in SERVICE_ACCOUNT_SUFFIX:
369    if change.author_email.endswith(suffix):
370      return []
371
372  results = []
373  at_least_one_docs_change = False
374  all_docs_changes = True
375  for affected_file in change.AffectedFiles():
376    affected_file_path = affected_file.LocalPath()
377    file_path, _ = os.path.splitext(affected_file_path)
378    if 'site' == file_path.split(os.path.sep)[0]:
379      at_least_one_docs_change = True
380    else:
381      all_docs_changes = False
382    if at_least_one_docs_change and not all_docs_changes:
383      break
384
385  footers = change.GitFootersFromDescription()
386  description_changed = False
387
388  # If the change includes only doc changes then add No-Try: true in the
389  # CL's description if it does not exist yet.
390  if all_docs_changes and 'true' not in footers.get('No-Try', []):
391    description_changed = True
392    change.AddDescriptionFooter('No-Try', 'true')
393    results.append(
394        output_api.PresubmitNotifyResult(
395            'This change has only doc changes. Automatically added '
396            '\'No-Try: true\' to the CL\'s description'))
397
398  # If the description has changed update it.
399  if description_changed:
400    gerrit.UpdateDescription(
401        change.FullDescriptionText(), change.issue)
402
403  return results
404
405
406def CheckChangeOnCommit(input_api, output_api):
407  """Presubmit checks for the change on commit."""
408  results = []
409  results.extend(_CommonChecks(input_api, output_api))
410  # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
411  # content of files.
412  results.extend(
413      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
414  return results
415