1# Copyright 2020 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Module used by CI tools in order to interact with fuzzers. This module helps 15CI tools to build fuzzers.""" 16 17import logging 18import os 19import sys 20 21import affected_fuzz_targets 22import continuous_integration 23import docker 24 25# pylint: disable=wrong-import-position,import-error 26sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 27import helper 28import utils 29 30# Default fuzz configuration. 31DEFAULT_ENGINE = 'libfuzzer' 32DEFAULT_ARCHITECTURE = 'x86_64' 33 34logging.basicConfig( 35 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 36 level=logging.DEBUG) 37 38 39def check_project_src_path(project_src_path): 40 """Returns True if |project_src_path| exists.""" 41 if not os.path.exists(project_src_path): 42 logging.error( 43 'PROJECT_SRC_PATH: %s does not exist. ' 44 'Are you mounting it correctly?', project_src_path) 45 return False 46 return True 47 48 49# pylint: disable=too-many-arguments 50 51 52class Builder: # pylint: disable=too-many-instance-attributes 53 """Class for fuzzer builders.""" 54 55 def __init__(self, config, ci_system): 56 self.config = config 57 self.ci_system = ci_system 58 self.out_dir = os.path.join(config.workspace, 'out') 59 os.makedirs(self.out_dir, exist_ok=True) 60 self.work_dir = os.path.join(config.workspace, 'work') 61 os.makedirs(self.work_dir, exist_ok=True) 62 self.image_repo_path = None 63 self.host_repo_path = None 64 self.repo_manager = None 65 66 def build_image_and_checkout_src(self): 67 """Builds the project builder image and checkout source code for the patch 68 we want to fuzz (if necessary). Returns True on success. 69 Must be implemented by child classes.""" 70 result = self.ci_system.prepare_for_fuzzer_build() 71 if not result.success: 72 return False 73 self.image_repo_path = result.image_repo_path 74 self.repo_manager = result.repo_manager 75 self.host_repo_path = self.repo_manager.repo_dir 76 return True 77 78 def build_fuzzers(self): 79 """Moves the source code we want to fuzz into the project builder and builds 80 the fuzzers from that source code. Returns True on success.""" 81 docker_args = get_common_docker_args(self.config.sanitizer, 82 self.config.language) 83 container = utils.get_container_name() 84 85 if container: 86 docker_args.extend( 87 _get_docker_build_fuzzers_args_container(self.out_dir, container)) 88 else: 89 docker_args.extend( 90 _get_docker_build_fuzzers_args_not_container(self.out_dir, 91 self.host_repo_path)) 92 93 if self.config.sanitizer == 'memory': 94 docker_args.extend(_get_docker_build_fuzzers_args_msan(self.work_dir)) 95 self.handle_msan_prebuild(container) 96 97 docker_args.extend([ 98 docker.get_project_image_name(self.config.project_name), 99 '/bin/bash', 100 '-c', 101 ]) 102 rm_path = os.path.join(self.image_repo_path, '*') 103 image_src_path = os.path.dirname(self.image_repo_path) 104 bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format( 105 rm_path, self.host_repo_path, image_src_path) 106 docker_args.append(bash_command) 107 logging.info('Building with %s sanitizer.', self.config.sanitizer) 108 if helper.docker_run(docker_args): 109 # docker_run returns nonzero on failure. 110 logging.error('Building fuzzers failed.') 111 return False 112 113 if self.config.sanitizer == 'memory': 114 self.handle_msan_postbuild(container) 115 return True 116 117 def handle_msan_postbuild(self, container): 118 """Post-build step for MSAN builds. Patches the build to use MSAN 119 libraries.""" 120 helper.docker_run([ 121 '--volumes-from', container, '-e', 122 'WORK={work_dir}'.format(work_dir=self.work_dir), 123 docker.MSAN_LIBS_BUILDER_TAG, 'patch_build.py', '/out' 124 ]) 125 126 def handle_msan_prebuild(self, container): 127 """Pre-build step for MSAN builds. Copies MSAN libs to |msan_libs_dir| and 128 returns docker arguments to use that directory for MSAN libs.""" 129 logging.info('Copying MSAN libs.') 130 helper.docker_run([ 131 '--volumes-from', container, docker.MSAN_LIBS_BUILDER_TAG, 'bash', '-c', 132 'cp -r /msan {work_dir}'.format(work_dir=self.work_dir) 133 ]) 134 135 def build(self): 136 """Builds the image, checkouts the source (if needed), builds the fuzzers 137 and then removes the unaffectted fuzzers. Returns True on success.""" 138 methods = [ 139 self.build_image_and_checkout_src, self.build_fuzzers, 140 self.remove_unaffected_fuzz_targets 141 ] 142 for method in methods: 143 if not method(): 144 return False 145 return True 146 147 def remove_unaffected_fuzz_targets(self): 148 """Removes the fuzzers unaffected by the patch.""" 149 if self.config.keep_unaffected_fuzz_targets: 150 logging.info('Not removing unaffected fuzz targets.') 151 return True 152 153 logging.info('Removing unaffected fuzz targets.') 154 changed_files = self.ci_system.get_changed_code_under_test( 155 self.repo_manager) 156 affected_fuzz_targets.remove_unaffected_fuzz_targets( 157 self.config.project_name, self.out_dir, changed_files, 158 self.image_repo_path) 159 return True 160 161 162def build_fuzzers(config): 163 """Builds all of the fuzzers for a specific OSS-Fuzz project. 164 165 Args: 166 project_name: The name of the OSS-Fuzz project being built. 167 project_repo_name: The name of the project's repo. 168 workspace: The location in a shared volume to store a git repo and build 169 artifacts. 170 pr_ref: The pull request reference to be built. 171 commit_sha: The commit sha for the project to be built at. 172 sanitizer: The sanitizer the fuzzers should be built with. 173 174 Returns: 175 True if build succeeded or False on failure. 176 """ 177 # Do some quick validation. 178 if config.project_src_path and not check_project_src_path( 179 config.project_src_path): 180 return False 181 182 # Get the builder and then build the fuzzers. 183 ci_system = continuous_integration.get_ci(config) 184 logging.info('ci_system: %s.', ci_system) 185 builder = Builder(config, ci_system) 186 return builder.build() 187 188 189def get_common_docker_args(sanitizer, language): 190 """Returns a list of common docker arguments.""" 191 return [ 192 '--cap-add', 193 'SYS_PTRACE', 194 '-e', 195 'FUZZING_ENGINE=' + DEFAULT_ENGINE, 196 '-e', 197 'SANITIZER=' + sanitizer, 198 '-e', 199 'ARCHITECTURE=' + DEFAULT_ARCHITECTURE, 200 '-e', 201 'CIFUZZ=True', 202 '-e', 203 'FUZZING_LANGUAGE=' + language, 204 ] 205 206 207def check_fuzzer_build(out_dir, 208 sanitizer, 209 language, 210 allowed_broken_targets_percentage=None): 211 """Checks the integrity of the built fuzzers. 212 213 Args: 214 out_dir: The directory containing the fuzzer binaries. 215 sanitizer: The sanitizer the fuzzers are built with. 216 217 Returns: 218 True if fuzzers are correct. 219 """ 220 if not os.path.exists(out_dir): 221 logging.error('Invalid out directory: %s.', out_dir) 222 return False 223 if not os.listdir(out_dir): 224 logging.error('No fuzzers found in out directory: %s.', out_dir) 225 return False 226 227 command = get_common_docker_args(sanitizer, language) 228 229 if allowed_broken_targets_percentage is not None: 230 command += [ 231 '-e', 232 ('ALLOWED_BROKEN_TARGETS_PERCENTAGE=' + 233 allowed_broken_targets_percentage) 234 ] 235 236 container = utils.get_container_name() 237 if container: 238 command += ['-e', 'OUT=' + out_dir, '--volumes-from', container] 239 else: 240 command += ['-v', '%s:/out' % out_dir] 241 command.extend(['-t', docker.BASE_RUNNER_TAG, 'test_all.py']) 242 exit_code = helper.docker_run(command) 243 logging.info('check fuzzer build exit code: %d', exit_code) 244 if exit_code: 245 logging.error('Check fuzzer build failed.') 246 return False 247 return True 248 249 250def _get_docker_build_fuzzers_args_container(host_out_dir, container): 251 """Returns arguments to the docker build arguments that are needed to use 252 |host_out_dir| when the host of the OSS-Fuzz builder container is another 253 container.""" 254 return ['-e', 'OUT=' + host_out_dir, '--volumes-from', container] 255 256 257def _get_docker_build_fuzzers_args_not_container(host_out_dir, host_repo_path): 258 """Returns arguments to the docker build arguments that are needed to use 259 |host_out_dir| when the host of the OSS-Fuzz builder container is not 260 another container.""" 261 image_out_dir = '/out' 262 return [ 263 '-e', 264 'OUT=' + image_out_dir, 265 '-v', 266 '%s:%s' % (host_out_dir, image_out_dir), 267 '-v', 268 '%s:%s' % (host_repo_path, host_repo_path), 269 ] 270 271 272def _get_docker_build_fuzzers_args_msan(work_dir): 273 """Returns arguments to the docker build command that are needed to use 274 MSAN.""" 275 # TODO(metzman): MSAN is broken, fix. 276 return [ 277 '-e', 'MSAN_LIBS_PATH={msan_libs_path}'.format( 278 msan_libs_path=os.path.join(work_dir, 'msan')) 279 ] 280