• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6"""Top-level presubmit script for Skia.
7
8See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
9for more details about the presubmit API built into gcl.
10"""
11
12import collections
13import csv
14import fnmatch
15import os
16import re
17import subprocess
18import sys
19import traceback
20
21
22REVERT_CL_SUBJECT_PREFIX = 'Revert '
23
24# Please add the complete email address here (and not just 'xyz@' or 'xyz').
25PUBLIC_API_OWNERS = (
26    'mtklein@google.com',
27    'reed@chromium.org',
28    'reed@google.com',
29    'bsalomon@chromium.org',
30    'bsalomon@google.com',
31    'djsollen@chromium.org',
32    'djsollen@google.com',
33    'hcm@chromium.org',
34    'hcm@google.com',
35)
36
37AUTHORS_FILE_NAME = 'AUTHORS'
38RELEASE_NOTES_FILE_NAME = 'RELEASE_NOTES.txt'
39
40DOCS_PREVIEW_URL = 'https://skia.org/?cl={issue}'
41GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
42
43SERVICE_ACCOUNT_SUFFIX = [
44    '@%s.iam.gserviceaccount.com' % project for project in [
45        'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public',
46        'skia-corp.google.com', 'chops-service-accounts']]
47
48
49def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
50  """Checks that files end with at least one \n (LF)."""
51  eof_files = []
52  for f in input_api.AffectedSourceFiles(source_file_filter):
53    contents = input_api.ReadFile(f, 'rb')
54    # Check that the file ends in at least one newline character.
55    if len(contents) > 1 and contents[-1:] != '\n':
56      eof_files.append(f.LocalPath())
57
58  if eof_files:
59    return [output_api.PresubmitPromptWarning(
60      'These files should end in a newline character:',
61      items=eof_files)]
62  return []
63
64
65def _JsonChecks(input_api, output_api):
66  """Run checks on any modified json files."""
67  failing_files = []
68  for affected_file in input_api.AffectedFiles(None):
69    affected_file_path = affected_file.LocalPath()
70    is_json = affected_file_path.endswith('.json')
71    is_metadata = (affected_file_path.startswith('site/') and
72                   affected_file_path.endswith('/METADATA'))
73    if is_json or is_metadata:
74      try:
75        input_api.json.load(open(affected_file_path, 'r'))
76      except ValueError:
77        failing_files.append(affected_file_path)
78
79  results = []
80  if failing_files:
81    results.append(
82        output_api.PresubmitError(
83            'The following files contain invalid json:\n%s\n\n' %
84                '\n'.join(failing_files)))
85  return results
86
87
88def _IfDefChecks(input_api, output_api):
89  """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
90  comment_block_start_pattern = re.compile('^\s*\/\*.*$')
91  comment_block_middle_pattern = re.compile('^\s+\*.*')
92  comment_block_end_pattern = re.compile('^\s+\*\/.*$')
93  single_line_comment_pattern = re.compile('^\s*//.*$')
94  def is_comment(line):
95    return (comment_block_start_pattern.match(line) or
96            comment_block_middle_pattern.match(line) or
97            comment_block_end_pattern.match(line) or
98            single_line_comment_pattern.match(line))
99
100  empty_line_pattern = re.compile('^\s*$')
101  def is_empty_line(line):
102    return empty_line_pattern.match(line)
103
104  failing_files = []
105  for affected_file in input_api.AffectedSourceFiles(None):
106    affected_file_path = affected_file.LocalPath()
107    if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
108      f = open(affected_file_path)
109      for line in f.xreadlines():
110        if is_comment(line) or is_empty_line(line):
111          continue
112        # The below will be the first real line after comments and newlines.
113        if line.startswith('#if 0 '):
114          pass
115        elif line.startswith('#if ') or line.startswith('#ifdef '):
116          failing_files.append(affected_file_path)
117        break
118
119  results = []
120  if failing_files:
121    results.append(
122        output_api.PresubmitError(
123            'The following files have #if or #ifdef before includes:\n%s\n\n'
124            'See https://bug.skia.org/3362 for why this should be fixed.' %
125                '\n'.join(failing_files)))
126  return results
127
128
129def _CopyrightChecks(input_api, output_api, source_file_filter=None):
130  results = []
131  year_pattern = r'\d{4}'
132  year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
133  years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
134  copyright_pattern = (
135      r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
136
137  for affected_file in input_api.AffectedSourceFiles(source_file_filter):
138    if 'third_party' in affected_file.LocalPath():
139      continue
140    contents = input_api.ReadFile(affected_file, 'rb')
141    if not re.search(copyright_pattern, contents):
142      results.append(output_api.PresubmitError(
143          '%s is missing a correct copyright header.' % affected_file))
144  return results
145
146
147def _InfraTests(input_api, output_api):
148  """Run the infra tests."""
149  results = []
150  if not any(f.LocalPath().startswith('infra')
151             for f in input_api.AffectedFiles()):
152    return results
153
154  cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
155  try:
156    subprocess.check_output(cmd)
157  except subprocess.CalledProcessError as e:
158    results.append(output_api.PresubmitError(
159        '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
160  return results
161
162
163def _CheckGNFormatted(input_api, output_api):
164  """Make sure any .gn files we're changing have been formatted."""
165  files = []
166  for f in input_api.AffectedFiles():
167    if (f.LocalPath().endswith('.gn') or
168        f.LocalPath().endswith('.gni')):
169      files.append(f)
170  if not files:
171    return []
172
173  cmd = ['python', os.path.join('bin', 'fetch-gn')]
174  try:
175    subprocess.check_output(cmd)
176  except subprocess.CalledProcessError as e:
177    return [output_api.PresubmitError(
178        '`%s` failed:\n%s' % (' '.join(cmd), e.output))]
179
180  results = []
181  for f in files:
182    gn = 'gn.exe' if 'win32' in sys.platform else 'gn'
183    gn = os.path.join(input_api.PresubmitLocalPath(), 'bin', gn)
184    cmd = [gn, 'format', '--dry-run', f.LocalPath()]
185    try:
186      subprocess.check_output(cmd)
187    except subprocess.CalledProcessError:
188      fix = 'bin/gn format ' + f.LocalPath()
189      results.append(output_api.PresubmitError(
190          '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
191  return results
192
193def _CheckIncludesFormatted(input_api, output_api):
194  """Make sure #includes in files we're changing have been formatted."""
195  files = [str(f) for f in input_api.AffectedFiles() if f.Action() != 'D']
196  cmd = ['python',
197         'tools/rewrite_includes.py',
198         '--dry-run'] + files
199  if 0 != subprocess.call(cmd):
200    return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))]
201  return []
202
203def _CheckCompileIsolate(input_api, output_api):
204  """Ensure that gen_compile_isolate.py does not change compile.isolate."""
205  # Only run the check if files were added or removed.
206  results = []
207  script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
208  isolate = os.path.join('infra', 'bots', 'compile.isolated')
209  for f in input_api.AffectedFiles():
210    if f.Action() in ('A', 'D', 'R'):
211      break
212    if f.LocalPath() in (script, isolate):
213      break
214  else:
215    return results
216
217  cmd = ['python', script, 'test']
218  try:
219    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
220  except subprocess.CalledProcessError as e:
221    results.append(output_api.PresubmitError(e.output))
222  return results
223
224
225class _WarningsAsErrors():
226  def __init__(self, output_api):
227    self.output_api = output_api
228    self.old_warning = None
229  def __enter__(self):
230    self.old_warning = self.output_api.PresubmitPromptWarning
231    self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError
232    return self.output_api
233  def __exit__(self, ex_type, ex_value, ex_traceback):
234    self.output_api.PresubmitPromptWarning = self.old_warning
235
236
237def _CheckDEPSValid(input_api, output_api):
238  """Ensure that DEPS contains valid entries."""
239  results = []
240  script = os.path.join('infra', 'bots', 'check_deps.py')
241  relevant_files = ('DEPS', script)
242  for f in input_api.AffectedFiles():
243    if f.LocalPath() in relevant_files:
244      break
245  else:
246    return results
247  cmd = ['python', script]
248  try:
249    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
250  except subprocess.CalledProcessError as e:
251    results.append(output_api.PresubmitError(e.output))
252  return results
253
254
255def _CommonChecks(input_api, output_api):
256  """Presubmit checks common to upload and commit."""
257  results = []
258  sources = lambda x: (x.LocalPath().endswith('.h') or
259                       x.LocalPath().endswith('.py') or
260                       x.LocalPath().endswith('.sh') or
261                       x.LocalPath().endswith('.m') or
262                       x.LocalPath().endswith('.mm') or
263                       x.LocalPath().endswith('.go') or
264                       x.LocalPath().endswith('.c') or
265                       x.LocalPath().endswith('.cc') or
266                       x.LocalPath().endswith('.cpp'))
267  results.extend(_CheckChangeHasEol(
268      input_api, output_api, source_file_filter=sources))
269  with _WarningsAsErrors(output_api):
270    results.extend(input_api.canned_checks.CheckChangeHasNoCR(
271        input_api, output_api, source_file_filter=sources))
272    results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
273        input_api, output_api, source_file_filter=sources))
274  results.extend(_JsonChecks(input_api, output_api))
275  results.extend(_IfDefChecks(input_api, output_api))
276  results.extend(_CopyrightChecks(input_api, output_api,
277                                  source_file_filter=sources))
278  results.extend(_CheckCompileIsolate(input_api, output_api))
279  results.extend(_CheckDEPSValid(input_api, output_api))
280  results.extend(_CheckIncludesFormatted(input_api, output_api))
281  return results
282
283
284def CheckChangeOnUpload(input_api, output_api):
285  """Presubmit checks for the change on upload.
286
287  The following are the presubmit checks:
288  * Check change has one and only one EOL.
289  """
290  results = []
291  results.extend(_CommonChecks(input_api, output_api))
292  # Run on upload, not commit, since the presubmit bot apparently doesn't have
293  # coverage or Go installed.
294  results.extend(_InfraTests(input_api, output_api))
295
296  results.extend(_CheckGNFormatted(input_api, output_api))
297  results.extend(_CheckReleaseNotesForPublicAPI(input_api, output_api))
298  return results
299
300
301class CodeReview(object):
302  """Abstracts which codereview tool is used for the specified issue."""
303
304  def __init__(self, input_api):
305    self._issue = input_api.change.issue
306    self._gerrit = input_api.gerrit
307
308  def GetOwnerEmail(self):
309    return self._gerrit.GetChangeOwner(self._issue)
310
311  def GetSubject(self):
312    return self._gerrit.GetChangeInfo(self._issue)['subject']
313
314  def GetDescription(self):
315    return self._gerrit.GetChangeDescription(self._issue)
316
317  def GetReviewers(self):
318    code_review_label = (
319        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
320    return [r['email'] for r in code_review_label.get('all', [])]
321
322  def GetApprovers(self):
323    approvers = []
324    code_review_label = (
325        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
326    for m in code_review_label.get('all', []):
327      if m.get("value") == 1:
328        approvers.append(m["email"])
329    return approvers
330
331
332def _CheckOwnerIsInAuthorsFile(input_api, output_api):
333  results = []
334  if input_api.change.issue:
335    cr = CodeReview(input_api)
336
337    owner_email = cr.GetOwnerEmail()
338
339    # Service accounts don't need to be in AUTHORS.
340    for suffix in SERVICE_ACCOUNT_SUFFIX:
341      if owner_email.endswith(suffix):
342        return results
343
344    try:
345      authors_content = ''
346      for line in open(AUTHORS_FILE_NAME):
347        if not line.startswith('#'):
348          authors_content += line
349      email_fnmatches = re.findall('<(.*)>', authors_content)
350      for email_fnmatch in email_fnmatches:
351        if fnmatch.fnmatch(owner_email, email_fnmatch):
352          # Found a match, the user is in the AUTHORS file break out of the loop
353          break
354      else:
355        results.append(
356          output_api.PresubmitError(
357            'The email %s is not in Skia\'s AUTHORS file.\n'
358            'Issue owner, this CL must include an addition to the Skia AUTHORS '
359            'file.'
360            % owner_email))
361    except IOError:
362      # Do not fail if authors file cannot be found.
363      traceback.print_exc()
364      input_api.logging.error('AUTHORS file not found!')
365
366  return results
367
368
369def _CheckReleaseNotesForPublicAPI(input_api, output_api):
370  """Checks to see if release notes file is updated with public API changes."""
371  results = []
372  public_api_changed = False
373  release_file_changed = False
374  for affected_file in input_api.AffectedFiles():
375    affected_file_path = affected_file.LocalPath()
376    file_path, file_ext = os.path.splitext(affected_file_path)
377    # We only care about files that end in .h and are under the top-level
378    # include dir, but not include/private.
379    if (file_ext == '.h' and
380        file_path.split(os.path.sep)[0] == 'include' and
381        'private' not in file_path):
382      public_api_changed = True
383    elif affected_file_path == RELEASE_NOTES_FILE_NAME:
384      release_file_changed = True
385
386  if public_api_changed and not release_file_changed:
387    results.append(output_api.PresubmitPromptWarning(
388        'If this change affects a client API, please add a summary line '
389        'to the %s file.' % RELEASE_NOTES_FILE_NAME))
390  return results
391
392
393
394def _CheckLGTMsForPublicAPI(input_api, output_api):
395  """Check LGTMs for public API changes.
396
397  For public API files make sure there is an LGTM from the list of owners in
398  PUBLIC_API_OWNERS.
399  """
400  results = []
401  requires_owner_check = False
402  for affected_file in input_api.AffectedFiles():
403    affected_file_path = affected_file.LocalPath()
404    file_path, file_ext = os.path.splitext(affected_file_path)
405    # We only care about files that end in .h and are under the top-level
406    # include dir, but not include/private.
407    if (file_ext == '.h' and
408        'include' == file_path.split(os.path.sep)[0] and
409        'private' not in file_path):
410      requires_owner_check = True
411
412  if not requires_owner_check:
413    return results
414
415  lgtm_from_owner = False
416  if input_api.change.issue:
417    cr = CodeReview(input_api)
418
419    if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
420      # It is a revert CL, ignore the public api owners check.
421      return results
422
423    if input_api.gerrit:
424      for reviewer in cr.GetReviewers():
425        if reviewer in PUBLIC_API_OWNERS:
426          # If an owner is specified as an reviewer in Gerrit then ignore the
427          # public api owners check.
428          return results
429    else:
430      match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
431      if match:
432        tbr_section = match.group(1).strip().split(' ')[0]
433        tbr_entries = tbr_section.split(',')
434        for owner in PUBLIC_API_OWNERS:
435          if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
436            # If an owner is specified in the TBR= line then ignore the public
437            # api owners check.
438            return results
439
440    if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
441      # An owner created the CL that is an automatic LGTM.
442      lgtm_from_owner = True
443
444    for approver in cr.GetApprovers():
445      if approver in PUBLIC_API_OWNERS:
446        # Found an lgtm in a message from an owner.
447        lgtm_from_owner = True
448        break
449
450  if not lgtm_from_owner:
451    results.append(
452        output_api.PresubmitError(
453            "If this CL adds to or changes Skia's public API, you need an LGTM "
454            "from any of %s.  If this CL only removes from or doesn't change "
455            "Skia's public API, please add a short note to the CL saying so. "
456            "Add one of the owners as a reviewer to your CL as well as to the "
457            "TBR= line.  If you don't know if this CL affects Skia's public "
458            "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
459  return results
460
461
462def PostUploadHook(gerrit, change, output_api):
463  """git cl upload will call this hook after the issue is created/modified.
464
465  This hook does the following:
466  * Adds a link to preview docs changes if there are any docs changes in the CL.
467  * Adds 'No-Try: true' if the CL contains only docs changes.
468  """
469  if not change.issue:
470    return []
471
472  # Skip PostUploadHooks for all auto-commit service account bots. New
473  # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
474  # the "--use-commit-queue" flag to "git cl upload".
475  for suffix in SERVICE_ACCOUNT_SUFFIX:
476    if change.author_email.endswith(suffix):
477      return []
478
479  results = []
480  at_least_one_docs_change = False
481  all_docs_changes = True
482  for affected_file in change.AffectedFiles():
483    affected_file_path = affected_file.LocalPath()
484    file_path, _ = os.path.splitext(affected_file_path)
485    if 'site' == file_path.split(os.path.sep)[0]:
486      at_least_one_docs_change = True
487    else:
488      all_docs_changes = False
489    if at_least_one_docs_change and not all_docs_changes:
490      break
491
492  footers = change.GitFootersFromDescription()
493  description_changed = False
494
495  # If the change includes only doc changes then add No-Try: true in the
496  # CL's description if it does not exist yet.
497  if all_docs_changes and 'true' not in footers.get('No-Try', []):
498    description_changed = True
499    change.AddDescriptionFooter('No-Try', 'true')
500    results.append(
501        output_api.PresubmitNotifyResult(
502            'This change has only doc changes. Automatically added '
503            '\'No-Try: true\' to the CL\'s description'))
504
505  # If there is at least one docs change then add preview link in the CL's
506  # description if it does not already exist there.
507  docs_preview_link = DOCS_PREVIEW_URL.format(issue=change.issue)
508  if (at_least_one_docs_change
509      and docs_preview_link not in footers.get('Docs-Preview', [])):
510    # Automatically add a link to where the docs can be previewed.
511    description_changed = True
512    change.AddDescriptionFooter('Docs-Preview', docs_preview_link)
513    results.append(
514        output_api.PresubmitNotifyResult(
515            'Automatically added a link to preview the docs changes to the '
516            'CL\'s description'))
517
518  # If the description has changed update it.
519  if description_changed:
520    gerrit.UpdateDescription(
521        change.FullDescriptionText(), change.issue)
522
523  return results
524
525
526def CheckChangeOnCommit(input_api, output_api):
527  """Presubmit checks for the change on commit.
528
529  The following are the presubmit checks:
530  * Check change has one and only one EOL.
531  * Ensures that the Skia tree is open in
532    http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
533    state and an error if it is in 'Closed' state.
534  """
535  results = []
536  results.extend(_CommonChecks(input_api, output_api))
537  results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
538  results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
539  # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
540  # content of files.
541  results.extend(
542      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
543  return results
544