1#!/usr/bin/env python3 2# Copyright 2020 Google LLC 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17"""Does bad_build_check on all fuzz targets in $OUT.""" 18 19import contextlib 20import multiprocessing 21import os 22import re 23import shutil 24import subprocess 25import stat 26import sys 27 28TMP_FUZZER_DIR = '/tmp/not-out' 29 30EXECUTABLE = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH 31 32IGNORED_TARGETS = [ 33 r'do_stuff_fuzzer', r'checksum_fuzzer', r'fuzz_dump', r'fuzz_keyring', 34 r'xmltest', r'fuzz_compression_sas_rle', r'ares_*_fuzzer' 35] 36 37IGNORED_TARGETS_RE = re.compile('^' + r'$|^'.join(IGNORED_TARGETS) + '$') 38 39 40def recreate_directory(directory): 41 """Creates |directory|. If it already exists than deletes it first before 42 creating.""" 43 if os.path.exists(directory): 44 shutil.rmtree(directory) 45 os.mkdir(directory) 46 47 48def move_directory_contents(src_directory, dst_directory): 49 """Moves contents of |src_directory| to |dst_directory|.""" 50 # Use mv because mv preserves file permissions. If we don't preserve file 51 # permissions that can mess up CheckFuzzerBuildTest in cifuzz_test.py and 52 # other cases where one is calling test_all on files not in OSS-Fuzz's real 53 # out directory. 54 src_contents = [ 55 os.path.join(src_directory, filename) 56 for filename in os.listdir(src_directory) 57 ] 58 command = ['mv'] + src_contents + [dst_directory] 59 subprocess.check_call(command) 60 61 62def is_elf(filepath): 63 """Returns True if |filepath| is an ELF file.""" 64 result = subprocess.run(['file', filepath], 65 stdout=subprocess.PIPE, 66 check=False) 67 return b'ELF' in result.stdout 68 69 70def find_fuzz_targets(directory, fuzzing_language): 71 """Returns paths to fuzz targets in |directory|.""" 72 # TODO(https://github.com/google/oss-fuzz/issues/4585): Use libClusterFuzz for 73 # this. 74 fuzz_targets = [] 75 for filename in os.listdir(directory): 76 path = os.path.join(directory, filename) 77 if filename == 'llvm-symbolizer': 78 continue 79 if filename.startswith('afl-'): 80 continue 81 if filename.startswith('jazzer_'): 82 continue 83 if not os.path.isfile(path): 84 continue 85 if not os.stat(path).st_mode & EXECUTABLE: 86 continue 87 # Fuzz targets are expected to be ELF binaries for languages other than 88 # Python and Java. 89 if (fuzzing_language != 'python' and fuzzing_language != 'jvm' and 90 not is_elf(path)): 91 continue 92 if os.getenv('FUZZING_ENGINE') != 'none': 93 with open(path, 'rb') as file_handle: 94 binary_contents = file_handle.read() 95 if b'LLVMFuzzerTestOneInput' not in binary_contents: 96 continue 97 fuzz_targets.append(path) 98 return fuzz_targets 99 100 101def do_bad_build_check(fuzz_target): 102 """Runs bad_build_check on |fuzz_target|. Returns a 103 Subprocess.ProcessResult.""" 104 print('INFO: performing bad build checks for', fuzz_target) 105 command = ['bad_build_check', fuzz_target] 106 return subprocess.run(command, 107 stderr=subprocess.PIPE, 108 stdout=subprocess.PIPE, 109 check=False) 110 111 112def get_broken_fuzz_targets(bad_build_results, fuzz_targets): 113 """Returns a list of broken fuzz targets and their process results in 114 |fuzz_targets| where each item in |bad_build_results| is the result of 115 bad_build_check on the corresponding element in |fuzz_targets|.""" 116 broken = [] 117 for result, fuzz_target in zip(bad_build_results, fuzz_targets): 118 if result.returncode != 0: 119 broken.append((fuzz_target, result)) 120 return broken 121 122 123def has_ignored_targets(out_dir): 124 """Returns True if |out_dir| has any fuzz targets we are supposed to ignore 125 bad build checks of.""" 126 out_files = set(os.listdir(out_dir)) 127 for filename in out_files: 128 if re.match(IGNORED_TARGETS_RE, filename): 129 return True 130 return False 131 132 133@contextlib.contextmanager 134def use_different_out_dir(): 135 """Context manager that moves OUT to TMP_FUZZER_DIR. This is useful for 136 catching hardcoding. Note that this sets the environment variable OUT and 137 therefore must be run before multiprocessing.Pool is created. Resets OUT at 138 the end.""" 139 # Use a fake OUT directory to catch path hardcoding that breaks on 140 # ClusterFuzz. 141 out = os.getenv('OUT') 142 initial_out = out 143 recreate_directory(TMP_FUZZER_DIR) 144 out = TMP_FUZZER_DIR 145 # Set this so that run_fuzzer which is called by bad_build_check works 146 # properly. 147 os.environ['OUT'] = out 148 # We move the contents of the directory because we can't move the 149 # directory itself because it is a mount. 150 move_directory_contents(initial_out, out) 151 try: 152 yield out 153 finally: 154 move_directory_contents(out, initial_out) 155 shutil.rmtree(out) 156 os.environ['OUT'] = initial_out 157 158 159def test_all_outside_out(fuzzing_language, allowed_broken_targets_percentage): 160 """Wrapper around test_all that changes OUT and returns the result.""" 161 with use_different_out_dir() as out: 162 return test_all(out, fuzzing_language, allowed_broken_targets_percentage) 163 164 165def test_all(out, fuzzing_language, allowed_broken_targets_percentage): 166 """Do bad_build_check on all fuzz targets.""" 167 # TODO(metzman): Refactor so that we can convert test_one to python. 168 fuzz_targets = find_fuzz_targets(out, fuzzing_language) 169 if not fuzz_targets: 170 print('ERROR: No fuzz targets found.') 171 return False 172 173 pool = multiprocessing.Pool() 174 bad_build_results = pool.map(do_bad_build_check, fuzz_targets) 175 broken_targets = get_broken_fuzz_targets(bad_build_results, fuzz_targets) 176 broken_targets_count = len(broken_targets) 177 if not broken_targets_count: 178 return True 179 180 print('Broken fuzz targets', broken_targets_count) 181 total_targets_count = len(fuzz_targets) 182 broken_targets_percentage = 100 * broken_targets_count / total_targets_count 183 for broken_target, result in broken_targets: 184 print(broken_target) 185 # Use write because we can't print binary strings. 186 sys.stdout.buffer.write(result.stdout + result.stderr + b'\n') 187 188 if broken_targets_percentage > allowed_broken_targets_percentage: 189 print('ERROR: {broken_targets_percentage}% of fuzz targets seem to be ' 190 'broken. See the list above for a detailed information.'.format( 191 broken_targets_percentage=broken_targets_percentage)) 192 if has_ignored_targets(out): 193 print('Build check automatically passing because of ignored targets.') 194 return True 195 return False 196 print('{total_targets_count} fuzzers total, {broken_targets_count} ' 197 'seem to be broken ({broken_targets_percentage}%).'.format( 198 total_targets_count=total_targets_count, 199 broken_targets_count=broken_targets_count, 200 broken_targets_percentage=broken_targets_percentage)) 201 return True 202 203 204def get_allowed_broken_targets_percentage(): 205 """Returns the value of the environment value 206 'ALLOWED_BROKEN_TARGETS_PERCENTAGE' as an int or returns a reasonable 207 default.""" 208 return int(os.getenv('ALLOWED_BROKEN_TARGETS_PERCENTAGE') or '10') 209 210 211def main(): 212 """Does bad_build_check on all fuzz targets in parallel. Returns 0 on success. 213 Returns 1 on failure.""" 214 # Set these environment variables here so that stdout 215 fuzzing_language = os.getenv('FUZZING_LANGUAGE') 216 allowed_broken_targets_percentage = get_allowed_broken_targets_percentage() 217 if not test_all_outside_out(fuzzing_language, 218 allowed_broken_targets_percentage): 219 return 1 220 return 0 221 222 223if __name__ == '__main__': 224 sys.exit(main()) 225