repohooks/rh/hooks.py

# -*- coding:utf-8 -*-
# Copyright 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Functions that implement the actual checks."""

from __future__ import print_function

import json
import os
import platform
import re
import sys

_path = os.path.realpath(__file__ + '/../..')
if sys.path[0] != _path:
    sys.path.insert(0, _path)
del _path

# pylint: disable=wrong-import-position
import rh.results
import rh.git
import rh.utils


class Placeholders(object):
    """Holder class for replacing ${vars} in arg lists.

    To add a new variable to replace in config files, just add it as a @property
    to this class using the form.  So to add support for BIRD:
      @property
      def var_BIRD(self):
        return <whatever this is>

    You can return either a string or an iterable (e.g. a list or tuple).
    """

    def __init__(self, diff=()):
        """Initialize.

        Args:
          diff: The list of files that changed.
        """
        self.diff = diff

    def expand_vars(self, args):
        """Perform place holder expansion on all of |args|.

        Args:
          args: The args to perform expansion on.

        Returns:
          The updated |args| list.
        """
        all_vars = set(self.vars())
        replacements = dict((var, self.get(var)) for var in all_vars)

        ret = []
        for arg in args:
            # First scan for exact matches
            for key, val in replacements.items():
                var = '${%s}' % (key,)
                if arg == var:
                    if isinstance(val, str):
                        ret.append(val)
                    else:
                        ret.extend(val)
                    # We break on first hit to avoid double expansion.
                    break
            else:
                # If no exact matches, do an inline replacement.
                def replace(m):
                    val = self.get(m.group(1))
                    if isinstance(val, str):
                        return val
                    else:
                        return ' '.join(val)
                ret.append(re.sub(r'\$\{(%s)\}' % ('|'.join(all_vars),),
                                  replace, arg))

        return ret

    @classmethod
    def vars(cls):
        """Yield all replacement variable names."""
        for key in dir(cls):
            if key.startswith('var_'):
                yield key[4:]

    def get(self, var):
        """Helper function to get the replacement |var| value."""
        return getattr(self, 'var_%s' % (var,))

    @property
    def var_PREUPLOAD_COMMIT_MESSAGE(self):
        """The git commit message."""
        return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '')

    @property
    def var_PREUPLOAD_COMMIT(self):
        """The git commit sha1."""
        return os.environ.get('PREUPLOAD_COMMIT', '')

    @property
    def var_PREUPLOAD_FILES(self):
        """List of files modified in this git commit."""
        return [x.file for x in self.diff if x.status != 'D']

    @property
    def var_REPO_ROOT(self):
        """The root of the repo checkout."""
        return rh.git.find_repo_root()

    @property
    def var_BUILD_OS(self):
        """The build OS (see _get_build_os_name for details)."""
        return _get_build_os_name()


class HookOptions(object):
    """Holder class for hook options."""

    def __init__(self, name, args, tool_paths):
        """Initialize.

        Args:
          name: The name of the hook.
          args: The override commandline arguments for the hook.
          tool_paths: A dictionary with tool names to paths.
        """
        self.name = name
        self._args = args
        self._tool_paths = tool_paths

    @staticmethod
    def expand_vars(args, diff=()):
        """Perform place holder expansion on all of |args|."""
        replacer = Placeholders(diff=diff)
        return replacer.expand_vars(args)

    def args(self, default_args=(), diff=()):
        """Gets the hook arguments, after performing place holder expansion.

        Args:
          default_args: The list to return if |self._args| is empty.
          diff: The list of files that changed in the current commit.

        Returns:
          A list with arguments.
        """
        args = self._args
        if not args:
            args = default_args

        return self.expand_vars(args, diff=diff)

    def tool_path(self, tool_name):
        """Gets the path in which the |tool_name| executable can be found.

        This function performs expansion for some place holders.  If the tool
        does not exist in the overridden |self._tool_paths| dictionary, the tool
        name will be returned and will be run from the user's $PATH.

        Args:
          tool_name: The name of the executable.

        Returns:
          The path of the tool with all optional place holders expanded.
        """
        assert tool_name in TOOL_PATHS
        if tool_name not in self._tool_paths:
            return TOOL_PATHS[tool_name]

        tool_path = os.path.normpath(self._tool_paths[tool_name])
        return self.expand_vars([tool_path])[0]


def _run_command(cmd, **kwargs):
    """Helper command for checks that tend to gather output."""
    kwargs.setdefault('redirect_stderr', True)
    kwargs.setdefault('combine_stdout_stderr', True)
    kwargs.setdefault('capture_output', True)
    kwargs.setdefault('error_code_ok', True)
    return rh.utils.run_command(cmd, **kwargs)


def _match_regex_list(subject, expressions):
    """Try to match a list of regular expressions to a string.

    Args:
      subject: The string to match regexes on.
      expressions: An iterable of regular expressions to check for matches with.

    Returns:
      Whether the passed in subject matches any of the passed in regexes.
    """
    for expr in expressions:
        if re.search(expr, subject):
            return True
    return False


def _filter_diff(diff, include_list, exclude_list=()):
    """Filter out files based on the conditions passed in.

    Args:
      diff: list of diff objects to filter.
      include_list: list of regex that when matched with a file path will cause
          it to be added to the output list unless the file is also matched with
          a regex in the exclude_list.
      exclude_list: list of regex that when matched with a file will prevent it
          from being added to the output list, even if it is also matched with a
          regex in the include_list.

    Returns:
      A list of filepaths that contain files matched in the include_list and not
      in the exclude_list.
    """
    filtered = []
    for d in diff:
        if (d.status != 'D' and
                _match_regex_list(d.file, include_list) and
                not _match_regex_list(d.file, exclude_list)):
            # We've got a match!
            filtered.append(d)
    return filtered


def _get_build_os_name():
    """Gets the build OS name.

    Returns:
      A string in a format usable to get prebuilt tool paths.
    """
    system = platform.system()
    if 'Darwin' in system or 'Macintosh' in system:
        return 'darwin-x86'
    else:
        # TODO: Add more values if needed.
        return 'linux-x86'


def _fixup_func_caller(cmd, **kwargs):
    """Wraps |cmd| around a callable automated fixup.

    For hooks that support automatically fixing errors after running (e.g. code
    formatters), this function provides a way to run |cmd| as the |fixup_func|
    parameter in HookCommandResult.
    """
    def wrapper():
        result = _run_command(cmd, **kwargs)
        if result.returncode not in (None, 0):
            return result.output
        return None
    return wrapper


def _check_cmd(hook_name, project, commit, cmd, fixup_func=None, **kwargs):
    """Runs |cmd| and returns its result as a HookCommandResult."""
    return [rh.results.HookCommandResult(hook_name, project, commit,
                                         _run_command(cmd, **kwargs),
                                         fixup_func=fixup_func)]


# Where helper programs exist.
TOOLS_DIR = os.path.realpath(__file__ + '/../../tools')

def get_helper_path(tool):
    """Return the full path to the helper |tool|."""
    return os.path.join(TOOLS_DIR, tool)


def check_custom(project, commit, _desc, diff, options=None, **kwargs):
    """Run a custom hook."""
    return _check_cmd(options.name, project, commit, options.args((), diff),
                      **kwargs)


def check_checkpatch(project, commit, _desc, diff, options=None):
    """Run |diff| through the kernel's checkpatch.pl tool."""
    tool = get_helper_path('checkpatch.pl')
    cmd = ([tool, '-', '--root', project.dir] +
           options.args(('--ignore=GERRIT_CHANGE_ID',), diff))
    return _check_cmd('checkpatch.pl', project, commit, cmd,
                      input=rh.git.get_patch(commit))


def check_clang_format(project, commit, _desc, diff, options=None):
    """Run git clang-format on the commit."""
    tool = get_helper_path('clang-format.py')
    clang_format = options.tool_path('clang-format')
    git_clang_format = options.tool_path('git-clang-format')
    tool_args = (['--clang-format', clang_format, '--git-clang-format',
                  git_clang_format] +
                 options.args(('--style', 'file', '--commit', commit), diff))
    cmd = [tool] + tool_args
    fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args)
    return _check_cmd('clang-format', project, commit, cmd,
                      fixup_func=fixup_func)


def check_google_java_format(project, commit, _desc, _diff, options=None):
    """Run google-java-format on the commit."""

    tool = get_helper_path('google-java-format.py')
    google_java_format = options.tool_path('google-java-format')
    google_java_format_diff = options.tool_path('google-java-format-diff')
    tool_args = ['--google-java-format', google_java_format,
                 '--google-java-format-diff', google_java_format_diff,
                 '--commit', commit] + options.args()
    cmd = [tool] + tool_args
    fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args)
    return _check_cmd('google-java-format', project, commit, cmd,
                      fixup_func=fixup_func)


def check_commit_msg_bug_field(project, commit, desc, _diff, options=None):
    """Check the commit message for a 'Bug:' line."""
    field = 'Bug'
    regex = r'^%s: (None|[0-9]+(, [0-9]+)*)$' % (field,)
    check_re = re.compile(regex)

    if options.args():
        raise ValueError('commit msg %s check takes no options' % (field,))

    found = []
    for line in desc.splitlines():
        if check_re.match(line):
            found.append(line)

    if not found:
        error = ('Commit message is missing a "%s:" line.  It must match the\n'
                 'following case-sensitive regex:\n\n    %s') % (field, regex)
    else:
        return

    return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
                                  project, commit, error=error)]


def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None):
    """Check the commit message for a 'Change-Id:' line."""
    field = 'Change-Id'
    regex = r'^%s: I[a-f0-9]+$' % (field,)
    check_re = re.compile(regex)

    if options.args():
        raise ValueError('commit msg %s check takes no options' % (field,))

    found = []
    for line in desc.splitlines():
        if check_re.match(line):
            found.append(line)

    if len(found) == 0:
        error = ('Commit message is missing a "%s:" line.  It must match the\n'
                 'following case-sensitive regex:\n\n    %s') % (field, regex)
    elif len(found) > 1:
        error = ('Commit message has too many "%s:" lines.  There can be only '
                 'one.') % (field,)
    else:
        return

    return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
                                  project, commit, error=error)]


PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK
information.  To generate the information, use the aapt tool to dump badging
information of the APKs being uploaded, specify where the APK was built, and
specify whether the APKs are suitable for release:

    for apk in $(find . -name '*.apk' | sort); do
        echo "${apk}"
        ${AAPT} dump badging "${apk}" |
            grep -iE "(package: |sdkVersion:|targetSdkVersion:)" |
            sed -e "s/' /'\\n/g"
        echo
    done

It must match the following case-sensitive multiline regex searches:

    %s

For more information, see go/platform-prebuilt and go/android-prebuilt.

"""


def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff,
                                         options=None):
    """Check that prebuilt APK commits contain the required lines."""

    if options.args():
        raise ValueError('prebuilt apk check takes no options')

    filtered = _filter_diff(diff, [r'\.apk$'])
    if not filtered:
        return

    regexes = [
        r'^package: .*$',
        r'^sdkVersion:.*$',
        r'^targetSdkVersion:.*$',
        r'^Built here:.*$',
        (r'^This build IS( NOT)? suitable for'
         r'( preview|( preview or)? public) release'
         r'( but IS NOT suitable for public release)?\.$')
    ]

    missing = []
    for regex in regexes:
        if not re.search(regex, desc, re.MULTILINE):
            missing.append(regex)

    if missing:
        error = PREBUILT_APK_MSG % '\n    '.join(missing)
    else:
        return

    return [rh.results.HookResult('commit msg: "prebuilt apk:" check',
                                  project, commit, error=error)]


TEST_MSG = """Commit message is missing a "Test:" line.  It must match the
following case-sensitive regex:

    %s

The Test: stanza is free-form and should describe how you tested your change.
As a CL author, you'll have a consistent place to describe the testing strategy
you use for your work. As a CL reviewer, you'll be reminded to discuss testing
as part of your code review, and you'll more easily replicate testing when you
patch in CLs locally.

Some examples below:

Test: make WITH_TIDY=1 mmma art
Test: make test-art
Test: manual - took a photo
Test: refactoring CL. Existing unit tests still pass.

Check the git history for more examples. It's a free-form field, so we urge
you to develop conventions that make sense for your project. Note that many
projects use exact test commands, which are perfectly fine.

Adding good automated tests with new code is critical to our goals of keeping
the system stable and constantly improving quality. Please use Test: to
highlight this area of your development. And reviewers, please insist on
high-quality Test: descriptions.
"""


def check_commit_msg_test_field(project, commit, desc, _diff, options=None):
    """Check the commit message for a 'Test:' line."""
    field = 'Test'
    regex = r'^%s: .*$' % (field,)
    check_re = re.compile(regex)

    if options.args():
        raise ValueError('commit msg %s check takes no options' % (field,))

    found = []
    for line in desc.splitlines():
        if check_re.match(line):
            found.append(line)

    if not found:
        error = TEST_MSG % (regex)
    else:
        return

    return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
                                  project, commit, error=error)]


def check_cpplint(project, commit, _desc, diff, options=None):
    """Run cpplint."""
    # This list matches what cpplint expects.  We could run on more (like .cxx),
    # but cpplint would just ignore them.
    filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$'])
    if not filtered:
        return

    cpplint = options.tool_path('cpplint')
    cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered)
    return _check_cmd('cpplint', project, commit, cmd)


def check_gofmt(project, commit, _desc, diff, options=None):
    """Checks that Go files are formatted with gofmt."""
    filtered = _filter_diff(diff, [r'\.go$'])
    if not filtered:
        return

    gofmt = options.tool_path('gofmt')
    cmd = [gofmt, '-l'] + options.args((), filtered)
    ret = []
    for d in filtered:
        data = rh.git.get_file_content(commit, d.file)
        result = _run_command(cmd, input=data)
        if result.output:
            ret.append(rh.results.HookResult(
                'gofmt', project, commit, error=result.output,
                files=(d.file,)))
    return ret


def check_json(project, commit, _desc, diff, options=None):
    """Verify json files are valid."""
    if options.args():
        raise ValueError('json check takes no options')

    filtered = _filter_diff(diff, [r'\.json$'])
    if not filtered:
        return

    ret = []
    for d in filtered:
        data = rh.git.get_file_content(commit, d.file)
        try:
            json.loads(data)
        except ValueError as e:
            ret.append(rh.results.HookResult(
                'json', project, commit, error=str(e),
                files=(d.file,)))
    return ret


def check_pylint(project, commit, _desc, diff, options=None):
    """Run pylint."""
    filtered = _filter_diff(diff, [r'\.py$'])
    if not filtered:
        return

    pylint = options.tool_path('pylint')
    cmd = [
        get_helper_path('pylint.py'),
        '--executable-path', pylint,
    ] + options.args(('${PREUPLOAD_FILES}',), filtered)
    return _check_cmd('pylint', project, commit, cmd)


def check_xmllint(project, commit, _desc, diff, options=None):
    """Run xmllint."""
    # XXX: Should we drop most of these and probe for <?xml> tags?
    extensions = frozenset((
        'dbus-xml',  # Generated DBUS interface.
        'dia',       # File format for Dia.
        'dtd',       # Document Type Definition.
        'fml',       # Fuzzy markup language.
        'form',      # Forms created by IntelliJ GUI Designer.
        'fxml',      # JavaFX user interfaces.
        'glade',     # Glade user interface design.
        'grd',       # GRIT translation files.
        'iml',       # Android build modules?
        'kml',       # Keyhole Markup Language.
        'mxml',      # Macromedia user interface markup language.
        'nib',       # OS X Cocoa Interface Builder.
        'plist',     # Property list (for OS X).
        'pom',       # Project Object Model (for Apache Maven).
        'rng',       # RELAX NG schemas.
        'sgml',      # Standard Generalized Markup Language.
        'svg',       # Scalable Vector Graphics.
        'uml',       # Unified Modeling Language.
        'vcproj',    # Microsoft Visual Studio project.
        'vcxproj',   # Microsoft Visual Studio project.
        'wxs',       # WiX Transform File.
        'xhtml',     # XML HTML.
        'xib',       # OS X Cocoa Interface Builder.
        'xlb',       # Android locale bundle.
        'xml',       # Extensible Markup Language.
        'xsd',       # XML Schema Definition.
        'xsl',       # Extensible Stylesheet Language.
    ))

    filtered = _filter_diff(diff, [r'\.(%s)$' % '|'.join(extensions)])
    if not filtered:
        return

    # TODO: Figure out how to integrate schema validation.
    # XXX: Should we use python's XML libs instead?
    cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered)

    return _check_cmd('xmllint', project, commit, cmd)


def check_android_test_mapping(project, commit, _desc, diff, options=None):
    """Verify Android TEST_MAPPING files are valid."""
    if options.args():
        raise ValueError('Android TEST_MAPPING check takes no options')
    filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$'])
    if not filtered:
        return

    testmapping_format = options.tool_path('android-test-mapping-format')
    cmd = [testmapping_format] + options.args(
        (project.dir, '${PREUPLOAD_FILES}',), filtered)
    return _check_cmd('android-test-mapping-format', project, commit, cmd)


# Hooks that projects can opt into.
# Note: Make sure to keep the top level README.md up to date when adding more!
BUILTIN_HOOKS = {
    'android_test_mapping_format': check_android_test_mapping,
    'checkpatch': check_checkpatch,
    'clang_format': check_clang_format,
    'commit_msg_bug_field': check_commit_msg_bug_field,
    'commit_msg_changeid_field': check_commit_msg_changeid_field,
    'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields,
    'commit_msg_test_field': check_commit_msg_test_field,
    'cpplint': check_cpplint,
    'gofmt': check_gofmt,
    'google_java_format': check_google_java_format,
    'jsonlint': check_json,
    'pylint': check_pylint,
    'xmllint': check_xmllint,
}

# Additional tools that the hooks can call with their default values.
# Note: Make sure to keep the top level README.md up to date when adding more!
TOOL_PATHS = {
    'android-test-mapping-format':
        os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'),
    'clang-format': 'clang-format',
    'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'),
    'git-clang-format': 'git-clang-format',
    'gofmt': 'gofmt',
    'google-java-format': 'google-java-format',
    'google-java-format-diff': 'google-java-format-diff.py',
    'pylint': 'pylint',
}