tools/repo_pull/repo_pull.py

#!/usr/bin/env python3

#
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""A command line utility to pull multiple change lists from Gerrit."""

from __future__ import print_function

import argparse
import collections
import itertools
import json
import multiprocessing
import os
import os.path
import re
import sys
import xml.dom.minidom

from gerrit import (
    create_url_opener_from_args, find_gerrit_name, normalize_gerrit_name,
    query_change_lists, run
)
from subprocess import PIPE

try:
    # pylint: disable=redefined-builtin
    from __builtin__ import raw_input as input  # PY2
except ImportError:
    pass

try:
    from shlex import quote as _sh_quote  # PY3.3
except ImportError:
    # Shell language simple string pattern.  If a string matches this pattern,
    # it doesn't have to be quoted.
    _SHELL_SIMPLE_PATTERN = re.compile('^[a-zA-Z90-9_./-]+$')

    def _sh_quote(txt):
        """Quote a string if it contains special characters."""
        return txt if _SHELL_SIMPLE_PATTERN.match(txt) else json.dumps(txt)


if bytes is str:
    def write_bytes(data, file):  # PY2
        """Write bytes to a file."""
        # pylint: disable=redefined-builtin
        file.write(data)
else:
    def write_bytes(data, file):  # PY3
        """Write bytes to a file."""
        # pylint: disable=redefined-builtin
        file.buffer.write(data)


def _confirm(question, default, file=sys.stderr):
    """Prompt a yes/no question and convert the answer to a boolean value."""
    # pylint: disable=redefined-builtin
    answers = {'': default, 'y': True, 'yes': True, 'n': False, 'no': False}
    suffix = '[Y/n] ' if default else ' [y/N] '
    while True:
        file.write(question + suffix)
        file.flush()
        ans = answers.get(input().lower())
        if ans is not None:
            return ans


class ChangeList(object):
    """A ChangeList to be checked out."""
    # pylint: disable=too-few-public-methods,too-many-instance-attributes

    def __init__(self, project, fetch, commit_sha1, commit, change_list):
        """Initialize a ChangeList instance."""
        # pylint: disable=too-many-arguments

        self.project = project
        self.number = change_list['_number']

        self.fetch = fetch

        fetch_git = None
        for protocol in ('http', 'sso', 'rpc'):
            fetch_git = fetch.get(protocol)
            if fetch_git:
                break

        if not fetch_git:
            raise ValueError(
                'unknown fetch protocols: ' + str(list(fetch.keys())))

        self.fetch_url = fetch_git['url']
        self.fetch_ref = fetch_git['ref']

        self.commit_sha1 = commit_sha1
        self.commit = commit
        self.parents = commit['parents']

        self.change_list = change_list


    def is_merge(self):
        """Check whether this change list a merge commit."""
        return len(self.parents) > 1


def find_repo_top(curdir):
    """Find the top directory for this git-repo source tree."""
    olddir = None
    while curdir != olddir:
        if os.path.exists(os.path.join(curdir, '.repo')):
            return curdir
        olddir = curdir
        curdir = os.path.dirname(curdir)
    raise ValueError('.repo dir not found')


def build_project_name_dir_dict(manifest_name):
    """Build the mapping from Gerrit project name to source tree project
    directory path."""
    manifest_cmd = ['repo', 'manifest']
    if manifest_name:
        manifest_cmd.extend(['-m', manifest_name])
    raw_manifest_xml = run(manifest_cmd, stdout=PIPE, check=True).stdout

    manifest_xml = xml.dom.minidom.parseString(raw_manifest_xml)
    project_dirs = {}
    for project in manifest_xml.getElementsByTagName('project'):
        name = project.getAttribute('name')
        path = project.getAttribute('path')
        if path:
            project_dirs[name] = path
        else:
            project_dirs[name] = name

    return project_dirs


def group_and_sort_change_lists(change_lists):
    """Build a dict that maps projects to a list of topologically sorted change
    lists."""

    # Build a dict that map projects to dicts that map commits to changes.
    projects = collections.defaultdict(dict)
    for change_list in change_lists:
        commit_sha1 = None
        for commit_sha1, value in change_list['revisions'].items():
            fetch = value['fetch']
            commit = value['commit']

        if not commit_sha1:
            raise ValueError('bad revision')

        project = change_list['project']

        project_changes = projects[project]
        if commit_sha1 in project_changes:
            raise KeyError('repeated commit sha1 "{}" in project "{}"'.format(
                commit_sha1, project))

        project_changes[commit_sha1] = ChangeList(
            project, fetch, commit_sha1, commit, change_list)

    # Sort all change lists in a project in post ordering.
    def _sort_project_change_lists(changes):
        visited_changes = set()
        sorted_changes = []

        def _post_order_traverse(change):
            visited_changes.add(change)
            for parent in change.parents:
                parent_change = changes.get(parent['commit'])
                if parent_change and parent_change not in visited_changes:
                    _post_order_traverse(parent_change)
            sorted_changes.append(change)

        for change in sorted(changes.values(), key=lambda x: x.number):
            if change not in visited_changes:
                _post_order_traverse(change)

        return sorted_changes

    # Sort changes in each projects
    sorted_changes = []
    for project in sorted(projects.keys()):
        sorted_changes.append(_sort_project_change_lists(projects[project]))

    return sorted_changes


def _main_json(args):
    """Print the change lists in JSON format."""
    change_lists = _get_change_lists_from_args(args)
    json.dump(change_lists, sys.stdout, indent=4, separators=(', ', ': '))
    print()  # Print the end-of-line


# Git commands for merge commits
_MERGE_COMMANDS = {
    'merge': ['git', 'merge', '--no-edit'],
    'merge-ff-only': ['git', 'merge', '--no-edit', '--ff-only'],
    'merge-no-ff': ['git', 'merge', '--no-edit', '--no-ff'],
    'reset': ['git', 'reset', '--hard'],
    'checkout': ['git', 'checkout'],
}


# Git commands for non-merge commits
_PICK_COMMANDS = {
    'pick': ['git', 'cherry-pick', '--allow-empty'],
    'merge': ['git', 'merge', '--no-edit'],
    'merge-ff-only': ['git', 'merge', '--no-edit', '--ff-only'],
    'merge-no-ff': ['git', 'merge', '--no-edit', '--no-ff'],
    'reset': ['git', 'reset', '--hard'],
    'checkout': ['git', 'checkout'],
}


def build_pull_commands(change, branch_name, merge_opt, pick_opt):
    """Build command lines for each change.  The command lines will be passed
    to subprocess.run()."""

    cmds = []
    if branch_name is not None:
        cmds.append(['repo', 'start', branch_name])
    cmds.append(['git', 'fetch', change.fetch_url, change.fetch_ref])
    if change.is_merge():
        cmds.append(_MERGE_COMMANDS[merge_opt] + ['FETCH_HEAD'])
    else:
        cmds.append(_PICK_COMMANDS[pick_opt] + ['FETCH_HEAD'])
    return cmds


def _sh_quote_command(cmd):
    """Convert a command (an argument to subprocess.run()) to a shell command
    string."""
    return ' '.join(_sh_quote(x) for x in cmd)


def _sh_quote_commands(cmds):
    """Convert multiple commands (arguments to subprocess.run()) to shell
    command strings."""
    return ' && '.join(_sh_quote_command(cmd) for cmd in cmds)


def _main_bash(args):
    """Print the bash command to pull the change lists."""
    repo_top = find_repo_top(os.getcwd())
    project_dirs = build_project_name_dir_dict(args.manifest)
    branch_name = _get_local_branch_name_from_args(args)

    change_lists = _get_change_lists_from_args(args)
    change_list_groups = group_and_sort_change_lists(change_lists)

    print(_sh_quote_command(['pushd', repo_top]))
    for changes in change_list_groups:
        for change in changes:
            project_dir = project_dirs.get(change.project, change.project)
            cmds = []
            cmds.append(['pushd', project_dir])
            cmds.extend(build_pull_commands(
                change, branch_name, args.merge, args.pick))
            cmds.append(['popd'])
            print(_sh_quote_commands(cmds))
    print(_sh_quote_command(['popd']))


def _do_pull_change_lists_for_project(task):
    """Pick a list of changes (usually under a project directory)."""
    changes, task_opts = task

    branch_name = task_opts['branch_name']
    merge_opt = task_opts['merge_opt']
    pick_opt = task_opts['pick_opt']
    project_dirs = task_opts['project_dirs']
    repo_top = task_opts['repo_top']

    for i, change in enumerate(changes):
        try:
            cwd = project_dirs[change.project]
        except KeyError:
            err_msg = 'error: project "{}" cannot be found in manifest.xml\n'
            err_msg = err_msg.format(change.project).encode('utf-8')
            return (change, changes[i + 1:], [], err_msg)

        print(change.commit_sha1[0:10], i + 1, cwd)
        cmds = build_pull_commands(change, branch_name, merge_opt, pick_opt)
        for cmd in cmds:
            proc = run(cmd, cwd=os.path.join(repo_top, cwd), stderr=PIPE)
            if proc.returncode != 0:
                return (change, changes[i + 1:], cmd, proc.stderr)
    return None


def _print_pull_failures(failures, file=sys.stderr):
    """Print pull failures and tracebacks."""
    # pylint: disable=redefined-builtin

    separator = '=' * 78
    separator_sub = '-' * 78

    print(separator, file=file)
    for failed_change, skipped_changes, cmd, errors in failures:
        print('PROJECT:', failed_change.project, file=file)
        print('FAILED COMMIT:', failed_change.commit_sha1, file=file)
        for change in skipped_changes:
            print('PENDING COMMIT:', change.commit_sha1, file=file)
        print(separator_sub, file=sys.stderr)
        print('FAILED COMMAND:', _sh_quote_command(cmd), file=file)
        write_bytes(errors, file=sys.stderr)
        print(separator, file=sys.stderr)


def _main_pull(args):
    """Pull the change lists."""
    repo_top = find_repo_top(os.getcwd())
    project_dirs = build_project_name_dir_dict(args.manifest)
    branch_name = _get_local_branch_name_from_args(args)

    # Collect change lists
    change_lists = _get_change_lists_from_args(args)
    change_list_groups = group_and_sort_change_lists(change_lists)

    # Build the options list for tasks
    task_opts = {
        'branch_name': branch_name,
        'merge_opt': args.merge,
        'pick_opt': args.pick,
        'project_dirs': project_dirs,
        'repo_top': repo_top,
    }

    # Run the commands to pull the change lists
    if args.parallel <= 1:
        results = [_do_pull_change_lists_for_project((changes, task_opts))
                   for changes in change_list_groups]
    else:
        pool = multiprocessing.Pool(processes=args.parallel)
        results = pool.map(_do_pull_change_lists_for_project,
                           zip(change_list_groups, itertools.repeat(task_opts)))

    # Print failures and tracebacks
    failures = [result for result in results if result]
    if failures:
        _print_pull_failures(failures)
        sys.exit(1)


def _parse_args():
    """Parse command line options."""
    parser = argparse.ArgumentParser()

    parser.add_argument('command', choices=['pull', 'bash', 'json'],
                        help='Commands')

    parser.add_argument('query', help='Change list query string')
    parser.add_argument('-g', '--gerrit', help='Gerrit review URL')

    parser.add_argument('--gitcookies',
                        default=os.path.expanduser('~/.gitcookies'),
                        help='Gerrit cookie file')
    parser.add_argument('--manifest', help='Manifest')
    parser.add_argument('--limits', default=1000, type=int,
                        help='Max number of change lists')
    parser.add_argument('--start', default=0, type=int,
                        help='Skip first N changes in query')

    parser.add_argument('-m', '--merge',
                        choices=sorted(_MERGE_COMMANDS.keys()),
                        default='merge-ff-only',
                        help='Method to pull merge commits')

    parser.add_argument('-p', '--pick',
                        choices=sorted(_PICK_COMMANDS.keys()),
                        default='pick',
                        help='Method to pull merge commits')

    parser.add_argument('-b', '--branch',
                        help='Local branch name for `repo start`')

    parser.add_argument('-j', '--parallel', default=1, type=int,
                        help='Number of parallel running commands')

    return parser.parse_args()


def _get_change_lists_from_args(args):
    """Query the change lists by args."""
    url_opener = create_url_opener_from_args(args)
    return query_change_lists(url_opener, args.gerrit, args.query, args.start,
                              args.limits)


def _get_local_branch_name_from_args(args):
    """Get the local branch name from args."""
    if not args.branch and not _confirm(
            'Do you want to continue without local branch name?', False):
        print('error: `-b` or `--branch` must be specified', file=sys.stderr)
        sys.exit(1)
    return args.branch


def main():
    """Main function"""
    args = _parse_args()

    if args.gerrit:
        args.gerrit = normalize_gerrit_name(args.gerrit)
    else:
        try:
            args.gerrit = find_gerrit_name()
        # pylint: disable=bare-except
        except:
            print('gerrit instance not found, use [-g GERRIT]')
            sys.exit(1)

    if args.command == 'json':
        _main_json(args)
    elif args.command == 'bash':
        _main_bash(args)
    elif args.command == 'pull':
        _main_pull(args)
    else:
        raise KeyError('unknown command')

if __name__ == '__main__':
    main()