1#!/usr/bin/env python3 2# Copyright 2020 Google LLC 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17"""Does bad_build_check on all fuzz targets in $OUT.""" 18 19import contextlib 20import multiprocessing 21import os 22import re 23import subprocess 24import stat 25import sys 26import tempfile 27 28BASE_TMP_FUZZER_DIR = '/tmp/not-out' 29 30EXECUTABLE = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH 31 32IGNORED_TARGETS = [ 33 r'do_stuff_fuzzer', r'checksum_fuzzer', r'fuzz_dump', r'fuzz_keyring', 34 r'xmltest', r'fuzz_compression_sas_rle', r'ares_*_fuzzer' 35] 36 37IGNORED_TARGETS_RE = re.compile('^' + r'$|^'.join(IGNORED_TARGETS) + '$') 38 39 40def move_directory_contents(src_directory, dst_directory): 41 """Moves contents of |src_directory| to |dst_directory|.""" 42 # Use mv because mv preserves file permissions. If we don't preserve file 43 # permissions that can mess up CheckFuzzerBuildTest in cifuzz_test.py and 44 # other cases where one is calling test_all on files not in OSS-Fuzz's real 45 # out directory. 46 src_contents = [ 47 os.path.join(src_directory, filename) 48 for filename in os.listdir(src_directory) 49 ] 50 command = ['mv'] + src_contents + [dst_directory] 51 subprocess.check_call(command) 52 53 54def is_elf(filepath): 55 """Returns True if |filepath| is an ELF file.""" 56 result = subprocess.run(['file', filepath], 57 stdout=subprocess.PIPE, 58 check=False) 59 return b'ELF' in result.stdout 60 61 62def is_shell_script(filepath): 63 """Returns True if |filepath| is a shell script.""" 64 result = subprocess.run(['file', filepath], 65 stdout=subprocess.PIPE, 66 check=False) 67 return b'shell script' in result.stdout 68 69 70def find_fuzz_targets(directory): 71 """Returns paths to fuzz targets in |directory|.""" 72 # TODO(https://github.com/google/oss-fuzz/issues/4585): Use libClusterFuzz for 73 # this. 74 fuzz_targets = [] 75 for filename in os.listdir(directory): 76 path = os.path.join(directory, filename) 77 if filename == 'llvm-symbolizer': 78 continue 79 if filename.startswith('afl-'): 80 continue 81 if filename.startswith('jazzer_'): 82 continue 83 if not os.path.isfile(path): 84 continue 85 if not os.stat(path).st_mode & EXECUTABLE: 86 continue 87 # Fuzz targets can either be ELF binaries or shell scripts (e.g. wrapper 88 # scripts for Python and JVM targets or rules_fuzzing builds with runfiles 89 # trees). 90 if not is_elf(path) and not is_shell_script(path): 91 continue 92 if os.getenv('FUZZING_ENGINE') != 'none': 93 with open(path, 'rb') as file_handle: 94 binary_contents = file_handle.read() 95 if b'LLVMFuzzerTestOneInput' not in binary_contents: 96 continue 97 fuzz_targets.append(path) 98 return fuzz_targets 99 100 101def do_bad_build_check(fuzz_target): 102 """Runs bad_build_check on |fuzz_target|. Returns a 103 Subprocess.ProcessResult.""" 104 print('INFO: performing bad build checks for', fuzz_target) 105 command = ['bad_build_check', fuzz_target] 106 return subprocess.run(command, 107 stderr=subprocess.PIPE, 108 stdout=subprocess.PIPE, 109 check=False) 110 111 112def get_broken_fuzz_targets(bad_build_results, fuzz_targets): 113 """Returns a list of broken fuzz targets and their process results in 114 |fuzz_targets| where each item in |bad_build_results| is the result of 115 bad_build_check on the corresponding element in |fuzz_targets|.""" 116 broken = [] 117 for result, fuzz_target in zip(bad_build_results, fuzz_targets): 118 if result.returncode != 0: 119 broken.append((fuzz_target, result)) 120 return broken 121 122 123def has_ignored_targets(out_dir): 124 """Returns True if |out_dir| has any fuzz targets we are supposed to ignore 125 bad build checks of.""" 126 out_files = set(os.listdir(out_dir)) 127 for filename in out_files: 128 if re.match(IGNORED_TARGETS_RE, filename): 129 return True 130 return False 131 132 133@contextlib.contextmanager 134def use_different_out_dir(): 135 """Context manager that moves OUT to subdirectory of BASE_TMP_FUZZER_DIR. This 136 is useful for catching hardcoding. Note that this sets the environment 137 variable OUT and therefore must be run before multiprocessing.Pool is created. 138 Resets OUT at the end.""" 139 # Use a fake OUT directory to catch path hardcoding that breaks on 140 # ClusterFuzz. 141 initial_out = os.getenv('OUT') 142 os.makedirs(BASE_TMP_FUZZER_DIR, exist_ok=True) 143 # Use a random subdirectory of BASE_TMP_FUZZER_DIR to allow running multiple 144 # instances of test_all in parallel (useful for integration testing). 145 with tempfile.TemporaryDirectory(dir=BASE_TMP_FUZZER_DIR) as out: 146 # Set this so that run_fuzzer which is called by bad_build_check works 147 # properly. 148 os.environ['OUT'] = out 149 # We move the contents of the directory because we can't move the 150 # directory itself because it is a mount. 151 move_directory_contents(initial_out, out) 152 try: 153 yield out 154 finally: 155 move_directory_contents(out, initial_out) 156 os.environ['OUT'] = initial_out 157 158 159def test_all_outside_out(allowed_broken_targets_percentage): 160 """Wrapper around test_all that changes OUT and returns the result.""" 161 with use_different_out_dir() as out: 162 return test_all(out, allowed_broken_targets_percentage) 163 164 165def test_all(out, allowed_broken_targets_percentage): 166 """Do bad_build_check on all fuzz targets.""" 167 # TODO(metzman): Refactor so that we can convert test_one to python. 168 fuzz_targets = find_fuzz_targets(out) 169 if not fuzz_targets: 170 print('ERROR: No fuzz targets found.') 171 return False 172 173 pool = multiprocessing.Pool() 174 bad_build_results = pool.map(do_bad_build_check, fuzz_targets) 175 pool.close() 176 pool.join() 177 broken_targets = get_broken_fuzz_targets(bad_build_results, fuzz_targets) 178 broken_targets_count = len(broken_targets) 179 if not broken_targets_count: 180 return True 181 182 print('Retrying failed fuzz targets sequentially', broken_targets_count) 183 pool = multiprocessing.Pool(1) 184 retry_targets = [] 185 for broken_target, result in broken_targets: 186 retry_targets.append(broken_target) 187 bad_build_results = pool.map(do_bad_build_check, retry_targets) 188 pool.close() 189 pool.join() 190 broken_targets = get_broken_fuzz_targets(bad_build_results, broken_targets) 191 broken_targets_count = len(broken_targets) 192 if not broken_targets_count: 193 return True 194 195 print('Broken fuzz targets', broken_targets_count) 196 total_targets_count = len(fuzz_targets) 197 broken_targets_percentage = 100 * broken_targets_count / total_targets_count 198 for broken_target, result in broken_targets: 199 print(broken_target) 200 # Use write because we can't print binary strings. 201 sys.stdout.buffer.write(result.stdout + result.stderr + b'\n') 202 203 if broken_targets_percentage > allowed_broken_targets_percentage: 204 print('ERROR: {broken_targets_percentage}% of fuzz targets seem to be ' 205 'broken. See the list above for a detailed information.'.format( 206 broken_targets_percentage=broken_targets_percentage)) 207 if has_ignored_targets(out): 208 print('Build check automatically passing because of ignored targets.') 209 return True 210 return False 211 print('{total_targets_count} fuzzers total, {broken_targets_count} ' 212 'seem to be broken ({broken_targets_percentage}%).'.format( 213 total_targets_count=total_targets_count, 214 broken_targets_count=broken_targets_count, 215 broken_targets_percentage=broken_targets_percentage)) 216 return True 217 218 219def get_allowed_broken_targets_percentage(): 220 """Returns the value of the environment value 221 'ALLOWED_BROKEN_TARGETS_PERCENTAGE' as an int or returns a reasonable 222 default.""" 223 return int(os.getenv('ALLOWED_BROKEN_TARGETS_PERCENTAGE') or '10') 224 225 226def main(): 227 """Does bad_build_check on all fuzz targets in parallel. Returns 0 on success. 228 Returns 1 on failure.""" 229 allowed_broken_targets_percentage = get_allowed_broken_targets_percentage() 230 if not test_all_outside_out(allowed_broken_targets_percentage): 231 return 1 232 return 0 233 234 235if __name__ == '__main__': 236 sys.exit(main()) 237