build/functions/build_project.py

# Copyright 2020 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
#!/usr/bin/env python3
"""Starts project build on Google Cloud Builder.

Usage: build_project.py <project_dir>
"""

from __future__ import print_function

import argparse
import collections
import datetime
import json
import logging
import os
import posixpath
import re
import sys

from googleapiclient.discovery import build as cloud_build
import oauth2client.client
import six
import yaml

import build_lib

FUZZING_BUILD_TYPE = 'fuzzing'

GCB_LOGS_BUCKET = 'oss-fuzz-gcb-logs'

DEFAULT_ARCHITECTURES = ['x86_64']
DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz']
DEFAULT_SANITIZERS = ['address', 'undefined']

LATEST_VERSION_FILENAME = 'latest.version'
LATEST_VERSION_CONTENT_TYPE = 'text/plain'

QUEUE_TTL_SECONDS = 60 * 60 * 24  # 24 hours.

PROJECTS_DIR = os.path.abspath(
    os.path.join(__file__, os.path.pardir, os.path.pardir, os.path.pardir,
                 os.path.pardir, 'projects'))

DEFAULT_GCB_OPTIONS = {'machineType': 'N1_HIGHCPU_32'}

Config = collections.namedtuple(
    'Config', ['testing', 'test_image_suffix', 'branch', 'parallel'])

WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')


class Build:  # pylint: disable=too-few-public-methods
  """Class representing the configuration for a build."""

  def __init__(self, fuzzing_engine, sanitizer, architecture):
    self.fuzzing_engine = fuzzing_engine
    self.sanitizer = sanitizer
    self.architecture = architecture
    self.targets_list_filename = build_lib.get_targets_list_filename(
        self.sanitizer)

  @property
  def out(self):
    """Returns the out directory for the build."""
    return posixpath.join(
        '/workspace/out/',
        f'{self.fuzzing_engine}-{self.sanitizer}-{self.architecture}')


def get_project_data(project_name):
  """Returns a tuple containing the contents of the project.yaml and Dockerfile
  of |project_name|. Raises a FileNotFoundError if there is no Dockerfile for
  |project_name|."""
  project_dir = os.path.join(PROJECTS_DIR, project_name)
  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
  try:
    with open(dockerfile_path) as dockerfile:
      dockerfile = dockerfile.read()
  except FileNotFoundError:
    logging.error('Project "%s" does not have a dockerfile.', project_name)
    raise
  project_yaml_path = os.path.join(project_dir, 'project.yaml')
  with open(project_yaml_path, 'r') as project_yaml_file_handle:
    project_yaml_contents = project_yaml_file_handle.read()
  return project_yaml_contents, dockerfile


class Project:  # pylint: disable=too-many-instance-attributes
  """Class representing an OSS-Fuzz project."""

  def __init__(self, name, project_yaml_contents, dockerfile, image_project):
    project_yaml = yaml.safe_load(project_yaml_contents)
    self.name = name
    self.image_project = image_project
    self.workdir = workdir_from_dockerfile(dockerfile)
    set_yaml_defaults(project_yaml)
    self._sanitizers = project_yaml['sanitizers']
    self.disabled = project_yaml['disabled']
    self.architectures = project_yaml['architectures']
    self.fuzzing_engines = project_yaml['fuzzing_engines']
    self.coverage_extra_args = project_yaml['coverage_extra_args']
    self.labels = project_yaml['labels']
    self.fuzzing_language = project_yaml['language']
    self.run_tests = project_yaml['run_tests']

  @property
  def sanitizers(self):
    """Returns processed sanitizers."""
    assert isinstance(self._sanitizers, list)
    processed_sanitizers = []
    for sanitizer in self._sanitizers:
      if isinstance(sanitizer, six.string_types):
        processed_sanitizers.append(sanitizer)
      elif isinstance(sanitizer, dict):
        for key in sanitizer.keys():
          processed_sanitizers.append(key)

    return processed_sanitizers

  @property
  def image(self):
    """Returns the docker image for the project."""
    return f'gcr.io/{self.image_project}/{self.name}'


def get_last_step_id(steps):
  """Returns the id of the last step in |steps|."""
  return steps[-1]['id']


def set_yaml_defaults(project_yaml):
  """Sets project.yaml's default parameters."""
  project_yaml.setdefault('disabled', False)
  project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES)
  project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
  project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
  project_yaml.setdefault('run_tests', True)
  project_yaml.setdefault('coverage_extra_args', '')
  project_yaml.setdefault('labels', {})


def is_supported_configuration(build):
  """Check if the given configuration is supported."""
  fuzzing_engine_info = build_lib.ENGINE_INFO[build.fuzzing_engine]
  if build.architecture == 'i386' and build.sanitizer != 'address':
    return False
  return (build.sanitizer in fuzzing_engine_info.supported_sanitizers and
          build.architecture in fuzzing_engine_info.supported_architectures)


def workdir_from_dockerfile(dockerfile):
  """Parses WORKDIR from the Dockerfile."""
  dockerfile_lines = dockerfile.split('\n')
  for line in dockerfile_lines:
    match = re.match(WORKDIR_REGEX, line)
    if match:
      # We need to escape '$' since they're used for subsitutions in Container
      # Builer builds.
      return match.group(1).replace('$', '$$')

  return '/src'


def get_datetime_now():
  """Returns datetime.datetime.now(). Used for mocking."""
  return datetime.datetime.now()


def get_env(fuzzing_language, build):
  """Returns an environment for building. The environment is returned as a list
  and is suitable for use as the "env" parameter in a GCB build step. The
  environment variables are based on the values of |fuzzing_language| and
  |build."""
  env_dict = {
      'FUZZING_LANGUAGE': fuzzing_language,
      'FUZZING_ENGINE': build.fuzzing_engine,
      'SANITIZER': build.sanitizer,
      'ARCHITECTURE': build.architecture,
      # Set HOME so that it doesn't point to a persisted volume (see
      # https://github.com/google/oss-fuzz/issues/6035).
      'HOME': '/root',
      'OUT': build.out,
  }
  return list(sorted([f'{key}={value}' for key, value in env_dict.items()]))


def get_compile_step(project, build, env, parallel):
  """Returns the GCB step for compiling |projects| fuzzers using |env|. The type
  of build is specified by |build|."""
  failure_msg = (
      '*' * 80 + '\nFailed to build.\nTo reproduce, run:\n'
      f'python infra/helper.py build_image {project.name}\n'
      'python infra/helper.py build_fuzzers --sanitizer '
      f'{build.sanitizer} --engine {build.fuzzing_engine} --architecture '
      f'{build.architecture} {project.name}\n' + '*' * 80)
  compile_step = {
      'name': project.image,
      'env': env,
      'args': [
          'bash',
          '-c',
          # Remove /out to make sure there are non instrumented binaries.
          # `cd /src && cd {workdir}` (where {workdir} is parsed from the
          # Dockerfile). Container Builder overrides our workdir so we need
          # to add this step to set it back.
          (f'rm -r /out && cd /src && cd {project.workdir} && '
           f'mkdir -p {build.out} && compile || '
           f'(echo "{failure_msg}" && false)'),
      ],
      'id': get_id('compile', build),
  }
  if parallel:
    maybe_add_parallel(compile_step, build_lib.get_srcmap_step_id(), parallel)
  return compile_step


def maybe_add_parallel(step, wait_for_id, parallel):
  """Makes |step| run immediately after |wait_for_id| if |parallel|. Mutates
  |step|."""
  if not parallel:
    return
  step['waitFor'] = wait_for_id


def get_id(step_type, build):
  """Returns a unique step id based on |step_type| and |build|. Useful for
  parallelizing builds."""
  return (f'{step_type}-{build.fuzzing_engine}-{build.sanitizer}'
          f'-{build.architecture}')


def get_build_steps(  # pylint: disable=too-many-locals, too-many-statements, too-many-branches, too-many-arguments
    project_name, project_yaml_contents, dockerfile, image_project,
    base_images_project, config):
  """Returns build steps for project."""

  project = Project(project_name, project_yaml_contents, dockerfile,
                    image_project)

  if project.disabled:
    logging.info('Project "%s" is disabled.', project.name)
    return []

  timestamp = get_datetime_now().strftime('%Y%m%d%H%M')

  build_steps = build_lib.project_image_steps(
      project.name,
      project.image,
      project.fuzzing_language,
      branch=config.branch,
      test_image_suffix=config.test_image_suffix)

  # Sort engines to make AFL first to test if libFuzzer has an advantage in
  # finding bugs first since it is generally built first.
  for fuzzing_engine in sorted(project.fuzzing_engines):
    for sanitizer in project.sanitizers:
      for architecture in project.architectures:
        build = Build(fuzzing_engine, sanitizer, architecture)
        if not is_supported_configuration(build):
          continue

        env = get_env(project.fuzzing_language, build)
        compile_step = get_compile_step(project, build, env, config.parallel)
        build_steps.append(compile_step)

        if project.run_tests:
          failure_msg = (
              '*' * 80 + '\nBuild checks failed.\n'
              'To reproduce, run:\n'
              f'python infra/helper.py build_image {project.name}\n'
              'python infra/helper.py build_fuzzers --sanitizer '
              f'{build.sanitizer} --engine {build.fuzzing_engine} '
              f'--architecture {build.architecture} {project.name}\n'
              'python infra/helper.py check_build --sanitizer '
              f'{build.sanitizer} --engine {build.fuzzing_engine} '
              f'--architecture {build.architecture} {project.name}\n' +
              '*' * 80)
          # Test fuzz targets.
          test_step = {
              'name':
                  get_runner_image_name(base_images_project,
                                        config.test_image_suffix),
              'env':
                  env,
              'args': [
                  'bash', '-c',
                  f'test_all.py || (echo "{failure_msg}" && false)'
              ],
              'id':
                  get_id('build-check', build)
          }
          maybe_add_parallel(test_step, get_last_step_id(build_steps),
                             config.parallel)
          build_steps.append(test_step)

        if project.labels:
          # Write target labels.
          build_steps.append({
              'name':
                  project.image,
              'env':
                  env,
              'args': [
                  '/usr/local/bin/write_labels.py',
                  json.dumps(project.labels),
                  build.out,
              ],
          })

        if build.sanitizer == 'dataflow' and build.fuzzing_engine == 'dataflow':
          dataflow_steps = dataflow_post_build_steps(project.name, env,
                                                     base_images_project,
                                                     config.testing,
                                                     config.test_image_suffix)
          if dataflow_steps:
            build_steps.extend(dataflow_steps)
          else:
            sys.stderr.write('Skipping dataflow post build steps.\n')

        build_steps.extend([
            # Generate targets list.
            {
                'name':
                    get_runner_image_name(base_images_project,
                                          config.test_image_suffix),
                'env':
                    env,
                'args': [
                    'bash', '-c',
                    f'targets_list > /workspace/{build.targets_list_filename}'
                ],
            }
        ])
        upload_steps = get_upload_steps(project, build, timestamp,
                                        base_images_project, config.testing)
        build_steps.extend(upload_steps)

  return build_steps


def get_targets_list_upload_step(bucket, project, build, uploader_image):
  """Returns the step to upload targets_list for |build| of |project| to
  |bucket|."""
  targets_list_url = build_lib.get_signed_url(
      build_lib.get_targets_list_url(bucket, project.name, build.sanitizer))
  return {
      'name': uploader_image,
      'args': [
          f'/workspace/{build.targets_list_filename}',
          targets_list_url,
      ],
  }


def get_uploader_image(base_images_project):
  """Returns the uploader base image in |base_images_project|."""
  return f'gcr.io/{base_images_project}/uploader'


def get_upload_steps(project, build, timestamp, base_images_project, testing):
  """Returns the steps for uploading the fuzzer build specified by |project| and
  |build|. Uses |timestamp| for naming the uploads. Uses |base_images_project|
  and |testing| for determining which image to use for the upload."""
  bucket = build_lib.get_upload_bucket(build.fuzzing_engine, build.architecture,
                                       testing)
  stamped_name = '-'.join([project.name, build.sanitizer, timestamp])
  zip_file = stamped_name + '.zip'
  upload_url = build_lib.get_signed_url(
      build_lib.GCS_UPLOAD_URL_FORMAT.format(bucket, project.name, zip_file))
  stamped_srcmap_file = stamped_name + '.srcmap.json'
  srcmap_url = build_lib.get_signed_url(
      build_lib.GCS_UPLOAD_URL_FORMAT.format(bucket, project.name,
                                             stamped_srcmap_file))
  latest_version_file = '-'.join(
      [project.name, build.sanitizer, LATEST_VERSION_FILENAME])
  latest_version_url = build_lib.GCS_UPLOAD_URL_FORMAT.format(
      bucket, project.name, latest_version_file)
  latest_version_url = build_lib.get_signed_url(
      latest_version_url, content_type=LATEST_VERSION_CONTENT_TYPE)
  uploader_image = get_uploader_image(base_images_project)

  upload_steps = [
      # Zip binaries.
      {
          'name': project.image,
          'args': ['bash', '-c', f'cd {build.out} && zip -r {zip_file} *'],
      },
      # Upload srcmap.
      {
          'name': uploader_image,
          'args': [
              '/workspace/srcmap.json',
              srcmap_url,
          ],
      },
      # Upload binaries.
      {
          'name': uploader_image,
          'args': [
              os.path.join(build.out, zip_file),
              upload_url,
          ],
      },
      # Upload targets list.
      get_targets_list_upload_step(bucket, project, build, uploader_image),
      # Upload the latest.version file.
      build_lib.http_upload_step(zip_file, latest_version_url,
                                 LATEST_VERSION_CONTENT_TYPE),
      # Cleanup.
      get_cleanup_step(project, build),
  ]
  return upload_steps


def get_cleanup_step(project, build):
  """Returns the step for cleaning up after doing |build| of |project|."""
  return {
      'name': project.image,
      'args': [
          'bash',
          '-c',
          'rm -r ' + build.out,
      ],
  }


def get_runner_image_name(base_images_project, test_image_suffix):
  """Returns the runner image that should be used, based on
  |base_images_project|. Returns the testing image if |test_image_suffix|."""
  image = f'gcr.io/{base_images_project}/base-runner'
  if test_image_suffix:
    image += '-' + test_image_suffix
  return image


def dataflow_post_build_steps(project_name, env, base_images_project, testing,
                              test_image_suffix):
  """Appends dataflow post build steps."""
  steps = build_lib.download_corpora_steps(project_name, testing)
  if not steps:
    return None

  steps.append({
      'name':
          get_runner_image_name(base_images_project, test_image_suffix),
      'env':
          env + [
              'COLLECT_DFT_TIMEOUT=2h',
              'DFT_FILE_SIZE_LIMIT=65535',
              'DFT_MIN_TIMEOUT=2.0',
              'DFT_TIMEOUT_RANGE=6.0',
          ],
      'args': [
          'bash', '-c',
          ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*}; done && '
           'collect_dft || (echo "DFT collection failed." && false)')
      ],
      'volumes': [{
          'name': 'corpus',
          'path': '/corpus'
      }],
  })
  return steps


def get_logs_url(build_id, cloud_project='oss-fuzz'):
  """Returns url where logs are displayed for the build."""
  return ('https://console.cloud.google.com/logs/viewer?'
          f'resource=build%2Fbuild_id%2F{build_id}&project={cloud_project}')


def get_gcb_url(build_id, cloud_project='oss-fuzz'):
  """Returns url where logs are displayed for the build."""
  return (f'https://console.cloud.google.com/cloud-build/builds/{build_id}'
          f'?project={cloud_project}')


# pylint: disable=no-member
def run_build(oss_fuzz_project,
              build_steps,
              credentials,
              build_type,
              cloud_project='oss-fuzz'):
  """Run the build for given steps on cloud build. |build_steps| are the steps
  to run. |credentials| are are used to authenticate to GCB and build in
  |cloud_project|. |oss_fuzz_project| and |build_type| are used to tag the build
  in GCB so the build can be queried for debugging purposes."""
  options = {}
  if 'GCB_OPTIONS' in os.environ:
    options = yaml.safe_load(os.environ['GCB_OPTIONS'])
  else:
    options = DEFAULT_GCB_OPTIONS

  tags = [oss_fuzz_project + '-' + build_type, build_type, oss_fuzz_project]
  build_body = {
      'steps': build_steps,
      'timeout': str(build_lib.BUILD_TIMEOUT) + 's',
      'options': options,
      'logsBucket': GCB_LOGS_BUCKET,
      'tags': tags,
      'queueTtl': str(QUEUE_TTL_SECONDS) + 's',
  }

  cloudbuild = cloud_build('cloudbuild',
                           'v1',
                           credentials=credentials,
                           cache_discovery=False)
  build_info = cloudbuild.projects().builds().create(projectId=cloud_project,
                                                     body=build_body).execute()
  build_id = build_info['metadata']['build']['id']

  logging.info('Build ID: %s', build_id)
  logging.info('Logs: %s', get_logs_url(build_id, cloud_project))
  logging.info('Cloud build page: %s', get_gcb_url(build_id, cloud_project))
  return build_id


def get_args(description):
  """Parses command line arguments and returns them. Suitable for a build
  script."""
  parser = argparse.ArgumentParser(sys.argv[0], description=description)
  parser.add_argument('projects', help='Projects.', nargs='+')
  parser.add_argument('--testing',
                      action='store_true',
                      required=False,
                      default=False,
                      help='Upload to testing buckets.')
  parser.add_argument('--test-image-suffix',
                      required=False,
                      default=None,
                      help='Use testing base-images.')
  parser.add_argument('--branch',
                      required=False,
                      default=None,
                      help='Use specified OSS-Fuzz branch.')
  parser.add_argument('--parallel',
                      action='store_true',
                      required=False,
                      default=False,
                      help='Do builds in parallel.')
  return parser.parse_args()


def build_script_main(script_description, get_build_steps_func, build_type):
  """Gets arguments from command line using |script_description| as helpstring
  description. Gets build_steps using |get_build_steps_func| and then runs those
  steps on GCB, tagging the builds with |build_type|. Returns 0 on success, 1 on
  failure."""
  args = get_args(script_description)
  logging.basicConfig(level=logging.INFO)

  image_project = 'oss-fuzz'
  base_images_project = 'oss-fuzz-base'

  credentials = oauth2client.client.GoogleCredentials.get_application_default()
  error = False
  config = Config(args.testing, args.test_image_suffix, args.branch,
                  args.parallel)
  for project_name in args.projects:
    logging.info('Getting steps for: "%s".', project_name)
    try:
      project_yaml_contents, dockerfile_contents = get_project_data(
          project_name)
    except FileNotFoundError:
      logging.error('Couldn\'t get project data. Skipping %s.', project_name)
      error = True
      continue

    steps = get_build_steps_func(project_name, project_yaml_contents,
                                 dockerfile_contents, image_project,
                                 base_images_project, config)
    if not steps:
      logging.error('No steps. Skipping %s.', project_name)
      error = True
      continue

    run_build(project_name, steps, credentials, build_type)
  return 0 if not error else 1


def main():
  """Build and run projects."""
  return build_script_main('Builds a project on GCB.', get_build_steps,
                           FUZZING_BUILD_TYPE)


if __name__ == '__main__':
  sys.exit(main())