infra/cifuzz/continuous_integration.py

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementations for various CI systems."""

import os
import collections
import sys
import logging

# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import build_specified_commit
import docker
import helper
import repo_manager
import retry
import utils
import workspace_utils

# pylint: disable=too-few-public-methods

BuildPreparationResult = collections.namedtuple(
    'BuildPreparationResult', ['success', 'image_repo_path', 'repo_manager'])

_IMAGE_BUILD_TRIES = 3
_IMAGE_BUILD_BACKOFF = 2


def fix_git_repo_for_diff(repo_manager_obj):
  """Fixes git repos cloned by the "checkout" action so that diffing works on
  them."""
  command = [
      'git', 'symbolic-ref', 'refs/remotes/origin/HEAD',
      'refs/remotes/origin/master'
  ]
  return utils.execute(command, location=repo_manager_obj.repo_dir)


class BaseCi:
  """Class representing common CI functionality."""

  def __init__(self, config):
    self.config = config
    self.workspace = workspace_utils.Workspace(config)

  def repo_dir(self):
    """Returns the source repo path, if it has been checked out. None is
    returned otherwise."""
    if not os.path.exists(self.workspace.repo_storage):
      return None

    # Note: this assumes there is only one repo checked out here.
    listing = os.listdir(self.workspace.repo_storage)
    if len(listing) != 1:
      raise RuntimeError('Invalid repo storage.')

    repo_path = os.path.join(self.workspace.repo_storage, listing[0])
    if not os.path.isdir(repo_path):
      raise RuntimeError('Repo is not a directory.')

    return repo_path

  def prepare_for_fuzzer_build(self):
    """Builds the fuzzer builder image and gets the source code we need to
    fuzz."""
    raise NotImplementedError('Child class must implement method.')

  def get_diff_base(self):
    """Returns the base to diff against with git to get the change under
    test."""
    raise NotImplementedError('Child class must implement method.')

  def get_changed_code_under_test(self, repo_manager_obj):
    """Returns the changed files that need to be tested."""
    base = self.get_diff_base()
    fix_git_repo_for_diff(repo_manager_obj)
    logging.info('Diffing against %s.', base)
    return repo_manager_obj.get_git_diff(base)

  def get_build_command(self, host_repo_path, image_repo_path):
    """Returns the command for building the project that is run inside the
    project builder container."""
    raise NotImplementedError('Child class must implement method.')


def get_build_command():
  """Returns the command to build the project inside the project builder
  container."""
  return 'compile'


def get_replace_repo_and_build_command(host_repo_path, image_repo_path):
  """Returns the command to replace the repo located at |image_repo_path| with
  |host_repo_path| and build the project inside the project builder
  container."""
  rm_path = os.path.join(image_repo_path, '*')
  image_src_path = os.path.dirname(image_repo_path)
  build_command = get_build_command()
  command = (f'cd / && rm -rf {rm_path} && cp -r {host_repo_path} '
             f'{image_src_path} && cd - && {build_command}')
  return command


def get_ci(config):
  """Determines what kind of CI is being used and returns the object
  representing that system."""

  if config.platform == config.Platform.EXTERNAL_GENERIC_CI:
    # Non-OSS-Fuzz projects must bring their own source and their own build
    # integration (which is relative to that source).
    return ExternalGeneric(config)
  if config.platform == config.Platform.EXTERNAL_GITHUB:
    # Non-OSS-Fuzz projects must bring their own source and their own build
    # integration (which is relative to that source).
    return ExternalGithub(config)

  if config.platform == config.Platform.INTERNAL_GENERIC_CI:
    # Builds of OSS-Fuzz projects not hosted on Github must bring their own
    # source since the checkout logic CIFuzz implements is github-specific.
    # TODO(metzman): Consider moving Github-actions builds of OSS-Fuzz projects
    # to this system to reduce implementation complexity.
    return InternalGeneric(config)

  return InternalGithub(config)


def checkout_specified_commit(repo_manager_obj, pr_ref, commit_sha):
  """Checks out the specified commit or pull request using
  |repo_manager_obj|."""
  try:
    if pr_ref:
      repo_manager_obj.checkout_pr(pr_ref)
    else:
      repo_manager_obj.checkout_commit(commit_sha)
  except (RuntimeError, ValueError):
    logging.error(
        'Can not check out requested state %s. '
        'Using current repo state', pr_ref or commit_sha)


class GithubCiMixin:
  """Mixin for Github based CI systems."""

  def get_diff_base(self):
    """Returns the base to diff against with git to get the change under
    test."""
    if self.config.base_ref:
      logging.debug('Diffing against base_ref: %s.', self.config.base_ref)
      return self.config.base_ref
    logging.debug('Diffing against base_commit: %s.', self.config.base_commit)
    return self.config.base_commit

  def get_changed_code_under_test(self, repo_manager_obj):
    """Returns the changed files that need to be tested."""
    if self.config.base_ref:
      repo_manager_obj.fetch_branch(self.config.base_ref)
    return super().get_changed_code_under_test(repo_manager_obj)


class InternalGithub(GithubCiMixin, BaseCi):
  """Class representing CI for an OSS-Fuzz project on Github Actions."""

  def prepare_for_fuzzer_build(self):
    """Builds the fuzzer builder image, checks out the pull request/commit and
    returns the BuildPreparationResult."""
    logging.info('Building OSS-Fuzz project on Github Actions.')
    assert self.config.pr_ref or self.config.commit_sha
    # detect_main_repo builds the image as a side effect.
    inferred_url, image_repo_path = (build_specified_commit.detect_main_repo(
        self.config.oss_fuzz_project_name,
        repo_name=self.config.project_repo_name))

    if not inferred_url or not image_repo_path:
      logging.error('Could not detect repo.')
      return BuildPreparationResult(success=False,
                                    image_repo_path=None,
                                    repo_manager=None)

    os.makedirs(self.workspace.repo_storage, exist_ok=True)

    # Use the same name used in the docker image so we can overwrite it.
    image_repo_name = os.path.basename(image_repo_path)

    # Checkout project's repo in the shared volume.
    manager = repo_manager.clone_repo_and_get_manager(
        inferred_url, self.workspace.repo_storage, repo_name=image_repo_name)
    checkout_specified_commit(manager, self.config.pr_ref,
                              self.config.commit_sha)

    return BuildPreparationResult(success=True,
                                  image_repo_path=image_repo_path,
                                  repo_manager=manager)

  def get_build_command(self, host_repo_path, image_repo_path):  # pylint: disable=no-self-use
    """Returns the command for building the project that is run inside the
    project builder container. Command also replaces |image_repo_path| with
    |host_repo_path|."""
    return get_replace_repo_and_build_command(host_repo_path, image_repo_path)


class InternalGeneric(BaseCi):
  """Class representing CI for an OSS-Fuzz project on a CI other than Github
  actions."""

  def prepare_for_fuzzer_build(self):
    """Builds the project builder image for an OSS-Fuzz project outside of
    GitHub actions. Returns the repo_manager. Does not checkout source code
    since external projects are expected to bring their own source code to
    CIFuzz."""
    logging.info('Building OSS-Fuzz project.')
    # detect_main_repo builds the image as a side effect.
    _, image_repo_path = (build_specified_commit.detect_main_repo(
        self.config.oss_fuzz_project_name,
        repo_name=self.config.project_repo_name))

    if not image_repo_path:
      logging.error('Could not detect repo.')
      return BuildPreparationResult(success=False,
                                    image_repo_path=None,
                                    repo_manager=None)

    manager = repo_manager.RepoManager(self.config.project_src_path)
    return BuildPreparationResult(success=True,
                                  image_repo_path=image_repo_path,
                                  repo_manager=manager)

  def get_diff_base(self):
    return 'origin...'

  def get_build_command(self, host_repo_path, image_repo_path):  # pylint: disable=no-self-use
    """Returns the command for building the project that is run inside the
    project builder container. Command also replaces |image_repo_path| with
    |host_repo_path|."""
    return get_replace_repo_and_build_command(host_repo_path, image_repo_path)


@retry.wrap(_IMAGE_BUILD_TRIES, _IMAGE_BUILD_BACKOFF)
def build_external_project_docker_image(project_src, build_integration_path):
  """Builds the project builder image for an external (non-OSS-Fuzz) project.
  Returns True on success."""
  dockerfile_path = os.path.join(build_integration_path, 'Dockerfile')
  command = [
      '-t', docker.EXTERNAL_PROJECT_IMAGE, '-f', dockerfile_path, project_src
  ]
  return helper.docker_build(command)


class ExternalGeneric(BaseCi):
  """CI implementation for generic CI for external (non-OSS-Fuzz) projects."""

  def get_diff_base(self):
    return 'origin...'

  def prepare_for_fuzzer_build(self):
    logging.info('ExternalGeneric: preparing for fuzzer build.')
    manager = repo_manager.RepoManager(self.config.project_src_path)
    build_integration_abs_path = os.path.join(
        manager.repo_dir, self.config.build_integration_path)
    if not build_external_project_docker_image(manager.repo_dir,
                                               build_integration_abs_path):
      logging.error('Failed to build external project: %s.',
                    self.config.oss_fuzz_project_name)
      return BuildPreparationResult(success=False,
                                    image_repo_path=None,
                                    repo_manager=None)

    image_repo_path = os.path.join('/src', self.config.project_repo_name)
    return BuildPreparationResult(success=True,
                                  image_repo_path=image_repo_path,
                                  repo_manager=manager)

  def get_build_command(self, host_repo_path, image_repo_path):  # pylint: disable=no-self-use
    """Returns the command for building the project that is run inside the
    project builder container."""
    return get_build_command()


class ExternalGithub(GithubCiMixin, BaseCi):
  """Class representing CI for a non-OSS-Fuzz project on Github Actions."""

  def prepare_for_fuzzer_build(self):
    """Builds the project builder image for a non-OSS-Fuzz project on GitHub
    actions. Sets the repo manager. Does not checkout source code since external
    projects are expected to bring their own source code to CIFuzz. Returns True
    on success."""
    logging.info('Building external project.')
    os.makedirs(self.workspace.repo_storage, exist_ok=True)
    # Checkout before building, so we don't need to rely on copying the source
    # into the image.
    # TODO(metzman): Figure out if we want second copy at all.
    manager = repo_manager.clone_repo_and_get_manager(
        self.config.git_url,
        self.workspace.repo_storage,
        repo_name=self.config.project_repo_name)
    checkout_specified_commit(manager, self.config.pr_ref,
                              self.config.commit_sha)

    build_integration_abs_path = os.path.join(
        manager.repo_dir, self.config.build_integration_path)
    if not build_external_project_docker_image(manager.repo_dir,
                                               build_integration_abs_path):
      logging.error('Failed to build external project.')
      return BuildPreparationResult(success=False,
                                    image_repo_path=None,
                                    repo_manager=None)

    image_repo_path = os.path.join('/src', self.config.project_repo_name)
    return BuildPreparationResult(success=True,
                                  image_repo_path=image_repo_path,
                                  repo_manager=manager)

  def get_build_command(self, host_repo_path, image_repo_path):  # pylint: disable=no-self-use
    """Returns the command for building the project that is run inside the
    project builder container."""
    return get_build_command()