• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2020 Google LLC
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17"""Does bad_build_check on all fuzz targets in $OUT."""
18
19import contextlib
20import multiprocessing
21import os
22import re
23import subprocess
24import stat
25import sys
26import tempfile
27
28BASE_TMP_FUZZER_DIR = '/tmp/not-out'
29
30EXECUTABLE = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
31
32IGNORED_TARGETS = [
33    r'do_stuff_fuzzer', r'checksum_fuzzer', r'fuzz_dump', r'fuzz_keyring',
34    r'xmltest', r'fuzz_compression_sas_rle', r'ares_*_fuzzer'
35]
36
37IGNORED_TARGETS_RE = re.compile('^' + r'$|^'.join(IGNORED_TARGETS) + '$')
38
39
40def move_directory_contents(src_directory, dst_directory):
41  """Moves contents of |src_directory| to |dst_directory|."""
42  # Use mv because mv preserves file permissions. If we don't preserve file
43  # permissions that can mess up CheckFuzzerBuildTest in cifuzz_test.py and
44  # other cases where one is calling test_all on files not in OSS-Fuzz's real
45  # out directory.
46  src_contents = [
47      os.path.join(src_directory, filename)
48      for filename in os.listdir(src_directory)
49  ]
50  command = ['mv'] + src_contents + [dst_directory]
51  subprocess.check_call(command)
52
53
54def is_elf(filepath):
55  """Returns True if |filepath| is an ELF file."""
56  result = subprocess.run(['file', filepath],
57                          stdout=subprocess.PIPE,
58                          check=False)
59  return b'ELF' in result.stdout
60
61
62def is_shell_script(filepath):
63  """Returns True if |filepath| is a shell script."""
64  result = subprocess.run(['file', filepath],
65                          stdout=subprocess.PIPE,
66                          check=False)
67  return b'shell script' in result.stdout
68
69
70def find_fuzz_targets(directory):
71  """Returns paths to fuzz targets in |directory|."""
72  # TODO(https://github.com/google/oss-fuzz/issues/4585): Use libClusterFuzz for
73  # this.
74  fuzz_targets = []
75  for filename in os.listdir(directory):
76    path = os.path.join(directory, filename)
77    if filename == 'llvm-symbolizer':
78      continue
79    if filename.startswith('afl-'):
80      continue
81    if filename.startswith('jazzer_'):
82      continue
83    if not os.path.isfile(path):
84      continue
85    if not os.stat(path).st_mode & EXECUTABLE:
86      continue
87    # Fuzz targets can either be ELF binaries or shell scripts (e.g. wrapper
88    # scripts for Python and JVM targets or rules_fuzzing builds with runfiles
89    # trees).
90    if not is_elf(path) and not is_shell_script(path):
91      continue
92    if os.getenv('FUZZING_ENGINE') != 'none':
93      with open(path, 'rb') as file_handle:
94        binary_contents = file_handle.read()
95        if b'LLVMFuzzerTestOneInput' not in binary_contents:
96          continue
97    fuzz_targets.append(path)
98  return fuzz_targets
99
100
101def do_bad_build_check(fuzz_target):
102  """Runs bad_build_check on |fuzz_target|. Returns a
103  Subprocess.ProcessResult."""
104  print('INFO: performing bad build checks for', fuzz_target)
105  command = ['bad_build_check', fuzz_target]
106  return subprocess.run(command,
107                        stderr=subprocess.PIPE,
108                        stdout=subprocess.PIPE,
109                        check=False)
110
111
112def get_broken_fuzz_targets(bad_build_results, fuzz_targets):
113  """Returns a list of broken fuzz targets and their process results in
114  |fuzz_targets| where each item in |bad_build_results| is the result of
115  bad_build_check on the corresponding element in |fuzz_targets|."""
116  broken = []
117  for result, fuzz_target in zip(bad_build_results, fuzz_targets):
118    if result.returncode != 0:
119      broken.append((fuzz_target, result))
120  return broken
121
122
123def has_ignored_targets(out_dir):
124  """Returns True if |out_dir| has any fuzz targets we are supposed to ignore
125  bad build checks of."""
126  out_files = set(os.listdir(out_dir))
127  for filename in out_files:
128    if re.match(IGNORED_TARGETS_RE, filename):
129      return True
130  return False
131
132
133@contextlib.contextmanager
134def use_different_out_dir():
135  """Context manager that moves OUT to subdirectory of BASE_TMP_FUZZER_DIR. This
136  is useful for catching hardcoding. Note that this sets the environment
137  variable OUT and therefore must be run before multiprocessing.Pool is created.
138  Resets OUT at the end."""
139  # Use a fake OUT directory to catch path hardcoding that breaks on
140  # ClusterFuzz.
141  initial_out = os.getenv('OUT')
142  os.makedirs(BASE_TMP_FUZZER_DIR, exist_ok=True)
143  # Use a random subdirectory of BASE_TMP_FUZZER_DIR to allow running multiple
144  # instances of test_all in parallel (useful for integration testing).
145  with tempfile.TemporaryDirectory(dir=BASE_TMP_FUZZER_DIR) as out:
146    # Set this so that run_fuzzer which is called by bad_build_check works
147    # properly.
148    os.environ['OUT'] = out
149    # We move the contents of the directory because we can't move the
150    # directory itself because it is a mount.
151    move_directory_contents(initial_out, out)
152    try:
153      yield out
154    finally:
155      move_directory_contents(out, initial_out)
156      os.environ['OUT'] = initial_out
157
158
159def test_all_outside_out(allowed_broken_targets_percentage):
160  """Wrapper around test_all that changes OUT and returns the result."""
161  with use_different_out_dir() as out:
162    return test_all(out, allowed_broken_targets_percentage)
163
164
165def test_all(out, allowed_broken_targets_percentage):
166  """Do bad_build_check on all fuzz targets."""
167  # TODO(metzman): Refactor so that we can convert test_one to python.
168  fuzz_targets = find_fuzz_targets(out)
169  if not fuzz_targets:
170    print('ERROR: No fuzz targets found.')
171    return False
172
173  pool = multiprocessing.Pool()
174  bad_build_results = pool.map(do_bad_build_check, fuzz_targets)
175  pool.close()
176  pool.join()
177  broken_targets = get_broken_fuzz_targets(bad_build_results, fuzz_targets)
178  broken_targets_count = len(broken_targets)
179  if not broken_targets_count:
180    return True
181
182  print('Retrying failed fuzz targets sequentially', broken_targets_count)
183  pool = multiprocessing.Pool(1)
184  retry_targets = []
185  for broken_target, result in broken_targets:
186    retry_targets.append(broken_target)
187  bad_build_results = pool.map(do_bad_build_check, retry_targets)
188  pool.close()
189  pool.join()
190  broken_targets = get_broken_fuzz_targets(bad_build_results, broken_targets)
191  broken_targets_count = len(broken_targets)
192  if not broken_targets_count:
193    return True
194
195  print('Broken fuzz targets', broken_targets_count)
196  total_targets_count = len(fuzz_targets)
197  broken_targets_percentage = 100 * broken_targets_count / total_targets_count
198  for broken_target, result in broken_targets:
199    print(broken_target)
200    # Use write because we can't print binary strings.
201    sys.stdout.buffer.write(result.stdout + result.stderr + b'\n')
202
203  if broken_targets_percentage > allowed_broken_targets_percentage:
204    print('ERROR: {broken_targets_percentage}% of fuzz targets seem to be '
205          'broken. See the list above for a detailed information.'.format(
206              broken_targets_percentage=broken_targets_percentage))
207    if has_ignored_targets(out):
208      print('Build check automatically passing because of ignored targets.')
209      return True
210    return False
211  print('{total_targets_count} fuzzers total, {broken_targets_count} '
212        'seem to be broken ({broken_targets_percentage}%).'.format(
213            total_targets_count=total_targets_count,
214            broken_targets_count=broken_targets_count,
215            broken_targets_percentage=broken_targets_percentage))
216  return True
217
218
219def get_allowed_broken_targets_percentage():
220  """Returns the value of the environment value
221  'ALLOWED_BROKEN_TARGETS_PERCENTAGE' as an int or returns a reasonable
222  default."""
223  return int(os.getenv('ALLOWED_BROKEN_TARGETS_PERCENTAGE') or '10')
224
225
226def main():
227  """Does bad_build_check on all fuzz targets in parallel. Returns 0 on success.
228  Returns 1 on failure."""
229  allowed_broken_targets_percentage = get_allowed_broken_targets_percentage()
230  if not test_all_outside_out(allowed_broken_targets_percentage):
231    return 1
232  return 0
233
234
235if __name__ == '__main__':
236  sys.exit(main())
237