• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Module used by CI tools in order to interact with fuzzers. This module helps
15CI tools to build fuzzers."""
16
17import logging
18import os
19import sys
20
21import affected_fuzz_targets
22import base_runner_utils
23import clusterfuzz_deployment
24import continuous_integration
25import docker
26import workspace_utils
27
28# pylint: disable=wrong-import-position,import-error
29sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
30import helper
31import utils
32
33logging.basicConfig(
34    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
35    level=logging.DEBUG)
36
37
38def check_project_src_path(project_src_path):
39  """Returns True if |project_src_path| exists."""
40  if not os.path.exists(project_src_path):
41    logging.error(
42        'PROJECT_SRC_PATH: %s does not exist. '
43        'Are you mounting it correctly?', project_src_path)
44    return False
45  return True
46
47
48# pylint: disable=too-many-arguments
49
50
51class Builder:  # pylint: disable=too-many-instance-attributes
52  """Class for fuzzer builders."""
53
54  def __init__(self, config, ci_system):
55    self.config = config
56    self.ci_system = ci_system
57    self.workspace = workspace_utils.Workspace(config)
58    self.workspace.initialize_dir(self.workspace.out)
59    self.workspace.initialize_dir(self.workspace.work)
60    self.clusterfuzz_deployment = (
61        clusterfuzz_deployment.get_clusterfuzz_deployment(
62            self.config, self.workspace))
63    self.image_repo_path = None
64    self.host_repo_path = None
65    self.repo_manager = None
66
67  def build_image_and_checkout_src(self):
68    """Builds the project builder image and checkout source code for the patch
69    we want to fuzz (if necessary). Returns True on success."""
70    result = self.ci_system.prepare_for_fuzzer_build()
71    if not result.success:
72      return False
73    self.image_repo_path = result.image_repo_path
74    self.repo_manager = result.repo_manager
75    logging.info('repo_dir: %s.', self.repo_manager.repo_dir)
76    self.host_repo_path = self.repo_manager.repo_dir
77    return True
78
79  def build_fuzzers(self):
80    """Moves the source code we want to fuzz into the project builder and builds
81    the fuzzers from that source code. Returns True on success."""
82    docker_args, docker_container = docker.get_base_docker_run_args(
83        self.workspace, self.config.sanitizer, self.config.language,
84        self.config.docker_in_docker)
85    if not docker_container:
86      docker_args.extend(
87          _get_docker_build_fuzzers_args_not_container(self.host_repo_path))
88
89    docker_args.extend([
90        docker.get_project_image_name(self.config.oss_fuzz_project_name),
91        '/bin/bash',
92        '-c',
93    ])
94    build_command = self.ci_system.get_build_command(self.host_repo_path,
95                                                     self.image_repo_path)
96    docker_args.append(build_command)
97    logging.info('Building with %s sanitizer.', self.config.sanitizer)
98
99    # TODO(metzman): Stop using helper.docker_run so we can get rid of
100    # docker.get_base_docker_run_args and merge its contents into
101    # docker.get_base_docker_run_command.
102    if not helper.docker_run(docker_args):
103      logging.error('Building fuzzers failed.')
104      return False
105
106    return True
107
108  def upload_build(self):
109    """Upload build."""
110    if self.config.upload_build:
111      self.clusterfuzz_deployment.upload_build(
112          self.repo_manager.get_current_commit())
113
114    return True
115
116  def check_fuzzer_build(self):
117    """Checks the fuzzer build. Returns True on success or if config specifies
118    to skip check."""
119    if not self.config.bad_build_check:
120      return True
121
122    return check_fuzzer_build(self.config)
123
124  def build(self):
125    """Builds the image, checkouts the source (if needed), builds the fuzzers
126    and then removes the unaffectted fuzzers. Returns True on success."""
127    methods = [
128        self.build_image_and_checkout_src,
129        self.build_fuzzers,
130        self.remove_unaffected_fuzz_targets,
131        self.check_fuzzer_build,
132        self.upload_build,
133    ]
134    for method in methods:
135      if not method():
136        return False
137    return True
138
139  def remove_unaffected_fuzz_targets(self):
140    """Removes the fuzzers unaffected by the patch."""
141    if self.config.keep_unaffected_fuzz_targets:
142      logging.info('Not removing unaffected fuzz targets.')
143      return True
144
145    logging.info('Removing unaffected fuzz targets.')
146    changed_files = self.ci_system.get_changed_code_under_test(
147        self.repo_manager)
148    affected_fuzz_targets.remove_unaffected_fuzz_targets(
149        self.clusterfuzz_deployment, self.workspace.out, changed_files,
150        self.image_repo_path)
151    return True
152
153
154def build_fuzzers(config):
155  """Builds all of the fuzzers for a specific OSS-Fuzz project.
156
157  Args:
158    project_name: The name of the OSS-Fuzz project being built.
159    project_repo_name: The name of the project's repo.
160    workspace: The location in a shared volume to store a git repo and build
161      artifacts.
162    pr_ref: The pull request reference to be built.
163    commit_sha: The commit sha for the project to be built at.
164    sanitizer: The sanitizer the fuzzers should be built with.
165
166  Returns:
167    True if build succeeded or False on failure.
168  """
169  # Do some quick validation.
170  if config.project_src_path and not check_project_src_path(
171      config.project_src_path):
172    return False
173
174  # Get the builder and then build the fuzzers.
175  ci_system = continuous_integration.get_ci(config)
176  logging.info('ci_system: %s.', ci_system)
177  builder = Builder(config, ci_system)
178  return builder.build()
179
180
181def check_fuzzer_build(config):
182  """Checks the integrity of the built fuzzers.
183
184  Args:
185    config: The config object.
186
187  Returns:
188    True if fuzzers pass OSS-Fuzz's build check.
189  """
190  workspace = workspace_utils.Workspace(config)
191  if not os.path.exists(workspace.out):
192    logging.error('Invalid out directory: %s.', workspace.out)
193    return False
194  if not os.listdir(workspace.out):
195    logging.error('No fuzzers found in out directory: %s.', workspace.out)
196    return False
197
198  env = base_runner_utils.get_env(config, workspace)
199  if config.allowed_broken_targets_percentage is not None:
200    env['ALLOWED_BROKEN_TARGETS_PERCENTAGE'] = (
201        config.allowed_broken_targets_percentage)
202
203  stdout, stderr, retcode = utils.execute('test_all.py', env=env)
204  print(f'Build check: stdout: {stdout}\nstderr: {stderr}')
205  if retcode == 0:
206    logging.info('Build check passed.')
207    return True
208  logging.error('Build check failed.')
209  return False
210
211
212def _get_docker_build_fuzzers_args_not_container(host_repo_path):
213  """Returns arguments to the docker build arguments that are needed to use
214  |host_repo_path| when the host of the OSS-Fuzz builder container is not
215  another container."""
216  return ['-v', f'{host_repo_path}:{host_repo_path}']
217