• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Implementations for various CI systems."""
15
16import os
17import collections
18import sys
19import logging
20
21# pylint: disable=wrong-import-position,import-error
22sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
23import build_specified_commit
24import helper
25import repo_manager
26import retry
27import utils
28
29# pylint: disable=too-few-public-methods
30
31BuildPreparationResult = collections.namedtuple(
32    'BuildPreparationResult', ['success', 'image_repo_path', 'repo_manager'])
33
34
35def fix_git_repo_for_diff(repo_manager_obj):
36  """Fixes git repos cloned by the "checkout" action so that diffing works on
37  them."""
38  command = [
39      'git', 'symbolic-ref', 'refs/remotes/origin/HEAD',
40      'refs/remotes/origin/master'
41  ]
42  return utils.execute(command, location=repo_manager_obj.repo_dir)
43
44
45class BaseCi:
46  """Class representing common CI functionality."""
47
48  def __init__(self, config):
49    self.config = config
50
51  def prepare_for_fuzzer_build(self):
52    """Builds the fuzzer builder image and gets the source code we need to
53    fuzz."""
54    raise NotImplementedError('Children must implement this method.')
55
56  def get_diff_base(self):
57    """Returns the base to diff against with git to get the change under
58    test."""
59    raise NotImplementedError('Children must implement this method.')
60
61  def get_changed_code_under_test(self, repo_manager_obj):
62    """Returns the changed files that need to be tested."""
63    base = self.get_diff_base()
64    fix_git_repo_for_diff(repo_manager_obj)
65    logging.info('Diffing against %s.', base)
66    return repo_manager_obj.get_git_diff(base)
67
68
69def get_ci(config):
70  """Determines what kind of CI is being used and returns the object
71  representing that system."""
72  if config.platform == config.Platform.EXTERNAL_GITHUB:
73    # Non-OSS-Fuzz projects must bring their own source and their own build
74    # integration (which is relative to that source).
75    return ExternalGithub(config)
76
77  if config.platform == config.Platform.INTERNAL_GENERIC_CI:
78    # Builds of OSS-Fuzz projects not hosted on Github must bring their own
79    # source since the checkout logic CIFuzz implements is github-specific.
80    # TODO(metzman): Consider moving Github-actions builds of OSS-Fuzz projects
81    # to this system to reduce implementation complexity.
82    return InternalGeneric(config)
83
84  return InternalGithub(config)
85
86
87def checkout_specified_commit(repo_manager_obj, pr_ref, commit_sha):
88  """Checks out the specified commit or pull request using
89  |repo_manager_obj|."""
90  try:
91    if pr_ref:
92      repo_manager_obj.checkout_pr(pr_ref)
93    else:
94      repo_manager_obj.checkout_commit(commit_sha)
95  except (RuntimeError, ValueError):
96    logging.error(
97        'Can not check out requested state %s. '
98        'Using current repo state', pr_ref or commit_sha)
99
100
101class GithubCiMixin:
102  """Mixin for Github based CI systems."""
103
104  def get_diff_base(self):
105    """Returns the base to diff against with git to get the change under
106    test."""
107    if self.config.base_ref:
108      logging.debug('Diffing against base_ref: %s.', self.config.base_ref)
109      return self.config.base_ref
110    logging.debug('Diffing against base_commit: %s.', self.config.base_commit)
111    return self.config.base_commit
112
113  def get_changed_code_under_test(self, repo_manager_obj):
114    """Returns the changed files that need to be tested."""
115    if self.config.base_ref:
116      repo_manager_obj.fetch_branch(self.config.base_ref)
117    return super().get_changed_code_under_test(repo_manager_obj)
118
119
120class InternalGithub(GithubCiMixin, BaseCi):
121  """Class representing CI for an OSS-Fuzz project on Github Actions."""
122
123  def prepare_for_fuzzer_build(self):
124    """Builds the fuzzer builder image, checks out the pull request/commit and
125    returns the BuildPreparationResult."""
126    logging.info('Building OSS-Fuzz project on Github Actions.')
127    assert self.config.pr_ref or self.config.commit_sha
128    # detect_main_repo builds the image as a side effect.
129    inferred_url, image_repo_path = (build_specified_commit.detect_main_repo(
130        self.config.project_name, repo_name=self.config.project_repo_name))
131
132    if not inferred_url or not image_repo_path:
133      logging.error('Could not detect repo from project %s.',
134                    self.config.project_name)
135      return BuildPreparationResult(False, None, None)
136
137    git_workspace = os.path.join(self.config.workspace, 'storage')
138    os.makedirs(git_workspace, exist_ok=True)
139
140    # Use the same name used in the docker image so we can overwrite it.
141    image_repo_name = os.path.basename(image_repo_path)
142
143    # Checkout project's repo in the shared volume.
144    manager = repo_manager.clone_repo_and_get_manager(inferred_url,
145                                                      git_workspace,
146                                                      repo_name=image_repo_name)
147    checkout_specified_commit(manager, self.config.pr_ref,
148                              self.config.commit_sha)
149
150    return BuildPreparationResult(True, image_repo_path, manager)
151
152
153class InternalGeneric(BaseCi):
154  """Class representing CI for an OSS-Fuzz project on a CI other than Github
155  actions."""
156
157  def prepare_for_fuzzer_build(self):
158    """Builds the project builder image for an OSS-Fuzz project outside of
159    GitHub actions. Returns the repo_manager. Does not checkout source code
160    since external projects are expected to bring their own source code to
161    CIFuzz."""
162    logging.info('Building OSS-Fuzz project.')
163    # detect_main_repo builds the image as a side effect.
164    _, image_repo_path = (build_specified_commit.detect_main_repo(
165        self.config.project_name, repo_name=self.config.project_repo_name))
166
167    if not image_repo_path:
168      logging.error('Could not detect repo from project %s.',
169                    self.config.project_name)
170      return BuildPreparationResult(False, None, None)
171
172    manager = repo_manager.RepoManager(self.config.project_src_path)
173    return BuildPreparationResult(True, image_repo_path, manager)
174
175  def get_diff_base(self):
176    return 'origin...'
177
178
179_IMAGE_BUILD_TRIES = 3
180_IMAGE_BUILD_BACKOFF = 2
181
182
183@retry.wrap(_IMAGE_BUILD_TRIES, _IMAGE_BUILD_BACKOFF)
184def build_external_project_docker_image(project_name, project_src,
185                                        build_integration_path):
186  """Builds the project builder image for an external (non-OSS-Fuzz) project.
187  Returns True on success."""
188  dockerfile_path = os.path.join(build_integration_path, 'Dockerfile')
189  tag = 'gcr.io/oss-fuzz/{project_name}'.format(project_name=project_name)
190  command = ['-t', tag, '-f', dockerfile_path, project_src]
191  return helper.docker_build(command)
192
193
194class ExternalGithub(GithubCiMixin, BaseCi):
195  """Class representing CI for a non-OSS-Fuzz project on Github Actions."""
196
197  def prepare_for_fuzzer_build(self):
198    """Builds the project builder image for a non-OSS-Fuzz project on GitHub
199    actions. Sets the repo manager. Does not checkout source code since external
200    projects are expected to bring their own source code to CIFuzz. Returns True
201    on success."""
202    logging.info('Building external project.')
203    git_workspace = os.path.join(self.config.workspace, 'storage')
204    os.makedirs(git_workspace, exist_ok=True)
205    # Checkout before building, so we don't need to rely on copying the source
206    # into the image.
207    # TODO(metzman): Figure out if we want second copy at all.
208    manager = repo_manager.clone_repo_and_get_manager(
209        self.config.git_url,
210        git_workspace,
211        repo_name=self.config.project_repo_name)
212    checkout_specified_commit(manager, self.config.pr_ref,
213                              self.config.commit_sha)
214
215    build_integration_path = os.path.join(manager.repo_dir,
216                                          self.config.build_integration_path)
217    if not build_external_project_docker_image(
218        self.config.project_name, manager.repo_dir, build_integration_path):
219      logging.error('Failed to build external project.')
220      return BuildPreparationResult(False, None, None)
221
222    image_repo_path = os.path.join('/src', self.config.project_repo_name)
223    return BuildPreparationResult(True, image_repo_path, manager)
224