autotest_lib/site_utils/automated_deploy.py

#!/usr/bin/python2

# Copyright (c) 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Module to automate the process of deploying to production.

Example usage of this script:
  1. Update both autotest and chromite to the lastest commit that has passed
     the test instance.
     $ ./site_utils/automated_deploy.py
  2. Skip updating a repo, e.g. autotest
     $ ./site_utils/automated_deploy.py --skip_autotest
  3. Update a given repo to a specific commit
     $ ./site_utils/automated_deploy.py --autotest_hash='1234'
"""

import argparse
import filecmp
import os
import re
import sys
import subprocess

import common
from autotest_lib.client.common_lib import revision_control
from autotest_lib.site_utils.lib import infra

INFRA_DATA = 'infradata'
AUTOTEST_DIR = common.autotest_dir
GIT_URL = {'autotest':
           'https://chromium.googlesource.com/chromiumos/third_party/autotest',
           'chromite':
           'https://chromium.googlesource.com/chromiumos/chromite',
           INFRA_DATA:
           'https://chrome-internal.googlesource.com/infradata/config'}
PROD_BRANCH = 'prod'
MASTER_AFE = 'cautotest'
NOTIFY_GROUP = 'chromeos-infra-discuss@google.com'

# CIPD packages whose prod refs should be updated.
_CIPD_PACKAGES = (
        'chromiumos/infra/lucifer',
        'chromiumos/infra/skylab/linux-amd64',
        'chromiumos/infra/skylab-inventory',
        'chromiumos/infra/skylab_swarming_worker/linux-amd64',
        'chromiumos/infra/autotest_status_parser/linux-amd64',
)


class AutoDeployException(Exception):
    """Raised when any deploy step fails."""


def parse_arguments():
    """Parse command line arguments.

    @returns An argparse.Namespace populated with argument values.
    """
    parser = argparse.ArgumentParser(
            description=('Command to update prod branch for autotest, chromite '
                         'repos. Then deploy new changes to all lab servers.'))
    parser.add_argument('--skip_autotest', action='store_true', default=False,
            help='Skip updating autotest prod branch. Default is False.')
    parser.add_argument('--skip_chromite', action='store_true', default=False,
            help='Skip updating chromite prod branch. Default is False.')
    parser.add_argument('--skip_prod_config_check', action='store_true',
                        default=False,
                        help='Skip checking prod config matches dev. '
                             'Default is False.')
    parser.add_argument('--force_update', action='store_true', default=False,
            help=('Force a deployment without updating both autotest and '
                  'chromite prod branch'))
    parser.add_argument('--autotest_hash', type=str, default=None,
            help='Update autotest prod branch to the given hash. If it is not'
                 ' specified, autotest prod branch will be rebased to '
                 'prod-next branch, which is the latest commit that has '
                 'passed our test instance.')
    parser.add_argument('--chromite_hash', type=str, default=None,
            help='Same as autotest_hash option.')

    results = parser.parse_args(sys.argv[1:])

    # Verify the validity of the options.
    if ((results.skip_autotest and results.autotest_hash) or
        (results.skip_chromite and results.chromite_hash)):
        parser.print_help()
        print 'Cannot specify skip_* and *_hash options at the same time.'
        sys.exit(1)
    if results.force_update:
      results.skip_autotest = True
      results.skip_chromite = True
    return results


def verify_dev_infradata_config_matches_prod():
    """Checks that dev config matches prod config before push

    Based on crbug.com/949696, this checks to make sure that dev
    config changes have been cloned to prod before a prod push.

    @raises subprocess.CalledProcessError on a command failure.
    @raised revision_control.GitCloneError when git clone fails.
    """

    repo_dir = os.path.join('/tmp', INFRA_DATA)
    git_url = GIT_URL[INFRA_DATA]
    print 'Checking PROD matches DEV config for %s repo' % git_url
    print 'Cloning %s master branch under %s' % (git_url, repo_dir)
    if os.path.exists(repo_dir):
        infra.local_runner('rm -rf %s' % repo_dir)
    git_repo = revision_control.GitRepo(repo_dir, git_url)
    git_repo.clone(shallow=True)

    dev_to_prod_files = {
        'configs/chromium-swarm-dev/scripts/skylab.py':
            'configs/chromeos-swarming/scripts/skylab.py',
    }

    for dev_rel_path, prod_rel_path in dev_to_prod_files.items():
        if filecmp.cmp(os.path.join(repo_dir, dev_rel_path),
                       os.path.join(repo_dir, prod_rel_path),
                       shallow=False):
            continue

        master_url = git_url + '/+/refs/heads/master/'
        raise AutoDeployException(
            '\n\n%s\nDOES NOT MATCH\n%s'
            '\n\nCopy DEV config to PROD config before performing release' %
            (master_url + prod_rel_path, master_url + dev_rel_path))

    print 'Successfully verified PROD config matches DEV config'


def clone_prod_branch(repo):
    """Method to clone the prod branch for a given repo under /tmp/ dir.

    @param repo: Name of the git repo to be cloned.

    @returns path to the cloned repo.
    @raises subprocess.CalledProcessError on a command failure.
    @raised revision_control.GitCloneError when git clone fails.
    """
    repo_dir = '/tmp/%s' % repo
    print 'Cloning %s prod branch under %s' % (repo, repo_dir)
    if os.path.exists(repo_dir):
        infra.local_runner('rm -rf %s' % repo_dir)
    git_repo = revision_control.GitRepo(repo_dir, GIT_URL[repo])
    git_repo.clone(remote_branch=PROD_BRANCH)
    print 'Successfully cloned %s prod branch' % repo
    return repo_dir


def update_prod_branch(repo, repo_dir, hash_to_rebase):
    """Method to update the prod branch of the given repo to the given hash.

    @param repo: Name of the git repo to be updated.
    @param repo_dir: path to the cloned repo.
    @param hash_to_rebase: Hash to rebase the prod branch to. If it is None,
                           prod branch will rebase to prod-next branch.

    @returns the range of the pushed commits as a string. E.g 123...345. If the
        prod branch is already up-to-date, return None.
    @raises subprocess.CalledProcessError on a command failure.
    """
    with infra.chdir(repo_dir):
        print 'Updating %s prod branch.' % repo
        rebase_to = hash_to_rebase if hash_to_rebase else 'origin/prod-next'
        # Check whether prod branch is already up-to-date, which means there is
        # no changes since last push.
        print 'Detecting new changes since last push...'
        diff = infra.local_runner('git log prod..%s --oneline' % rebase_to,
                                  stream_output=True)
        if diff:
            print 'Find new changes, will update prod branch...'
            infra.local_runner('git rebase %s prod' % rebase_to,
                               stream_output=True)
            result = infra.local_runner('git push origin prod',
                                        stream_output=True)
            print 'Successfully pushed %s prod branch!\n' % repo

            # Get the pushed commit range, which is used to get pushed commits
            # using git log E.g. 123..456, then run git log --oneline 123..456.
            grep = re.search('(\w)*\.\.(\w)*', result)

            if not grep:
                raise AutoDeployException(
                    'Fail to get pushed commits for repo %s from git log: %s' %
                    (repo, result))
            return grep.group(0)
        else:
            print 'No new %s changes found since last push.' % repo
            return None


def get_pushed_commits(repo, repo_dir, pushed_commits_range):
    """Method to get the pushed commits.

    @param repo: Name of the updated git repo.
    @param repo_dir: path to the cloned repo.
    @param pushed_commits_range: The range of the pushed commits. E.g 123...345
    @return: the commits that are pushed to prod branch. The format likes this:
             "git log --oneline A...B | grep autotest
              A xxxx
              B xxxx"
    @raises subprocess.CalledProcessError on a command failure.
    """
    print 'Getting pushed CLs for %s repo.' % repo
    if not pushed_commits_range:
        return '\n%s:\nNo new changes since last push.' % repo

    with infra.chdir(repo_dir):
        get_commits_cmd = 'git log --oneline %s' % pushed_commits_range

        pushed_commits = infra.local_runner(
                get_commits_cmd, stream_output=True)
        if repo == 'autotest':
            autotest_commits = ''
            for cl in pushed_commits.splitlines():
                if 'autotest' in cl:
                    autotest_commits += '%s\n' % cl

            pushed_commits = autotest_commits

        print 'Successfully got pushed CLs for %s repo!\n' % repo
        displayed_cmd = get_commits_cmd
        if repo == 'autotest':
          displayed_cmd += ' | grep autotest'
        return '\n%s:\n%s\n%s\n' % (repo, displayed_cmd, pushed_commits)


def kick_off_deploy():
    """Method to kick off deploy script to deploy changes to lab servers.

    @raises subprocess.CalledProcessError on a repo command failure.
    """
    print 'Start deploying changes to all lab servers...'
    with infra.chdir(AUTOTEST_DIR):
        # Then kick off the deploy script.
        deploy_cmd = ('runlocalssh ./site_utils/deploy_server.py -x --afe=%s' %
                      MASTER_AFE)
        infra.local_runner(deploy_cmd, stream_output=True)
        print 'Successfully deployed changes to all lab servers.'


def main(args):
    """Main entry"""
    options = parse_arguments()
    repos = dict()
    if not options.skip_autotest:
        repos.update({'autotest': options.autotest_hash})
    if not options.skip_chromite:
        repos.update({'chromite': options.chromite_hash})

    if not options.skip_prod_config_check:
        verify_dev_infradata_config_matches_prod()

    print 'Moving CIPD prod refs to prod-next'
    for pkg in _CIPD_PACKAGES:
        subprocess.check_call(['cipd', 'set-ref', pkg, '-version', 'prod-next',
                               '-ref', 'prod'])
    try:
        # update_log saves the git log of the updated repo.
        update_log = ''
        for repo, hash_to_rebase in repos.iteritems():
            repo_dir = clone_prod_branch(repo)
            push_commits_range = update_prod_branch(
                repo, repo_dir, hash_to_rebase)
            update_log += get_pushed_commits(repo, repo_dir, push_commits_range)

        kick_off_deploy()
    except revision_control.GitCloneError as e:
        print 'Fail to clone prod branch. Error:\n%s\n' % e
        raise
    except subprocess.CalledProcessError as e:
        print ('Deploy fails when running a subprocess cmd :\n%s\n'
               'Below is the push log:\n%s\n' % (e.output, update_log))
        raise
    except Exception as e:
        print 'Deploy fails with error:\n%s\nPush log:\n%s\n' % (e, update_log)
        raise

    # When deploy succeeds, print the update_log.
    print ('Deploy succeeds!!! Below is the push log of the updated repo:\n%s'
           'Please email this to %s.'% (update_log, NOTIFY_GROUP))


if __name__ == '__main__':
    sys.exit(main(sys.argv))