1#!/bin/bash -u 2# Copyright 2017 Google Inc. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17 18# A minimal number of runs to test fuzz target with a non-empty input. 19MIN_NUMBER_OF_RUNS=4 20 21# The "example" target has 73 with ASan, 65 with UBSan, and 6648 with MSan. 22# Real world targets have greater values (arduinojson: 407, zlib: 664). 23# Mercurial's bdiff_fuzzer has 116 PCs when built with ASan. 24THRESHOLD_FOR_NUMBER_OF_EDGES=100 25 26# A fuzz target is supposed to have at least two functions, such as 27# LLVMFuzzerTestOneInput and an API that is being called from there. 28THRESHOLD_FOR_NUMBER_OF_FUNCTIONS=2 29 30# Threshold values for different sanitizers used by instrumentation checks. 31ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000 32ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0 33 34# The value below can definitely be higher (like 500-1000), but avoid being too 35# agressive here while still evaluating the DFT-based fuzzing approach. 36DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD=100 37DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD=0 38 39MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000 40# Some engines (e.g. honggfuzz) may make a very small number of calls to msan 41# for memory poisoning. 42MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=3 43 44# Usually, a non UBSan build (e.g. ASan) has 165 calls to UBSan runtime. The 45# majority of targets built with UBSan have 200+ UBSan calls, but there are 46# some very small targets that may have < 200 UBSan calls even in a UBSan build. 47# Use the threshold value of 169 (slightly > 165) for UBSan build. 48UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=169 49 50# It would be risky to use the threshold value close to 165 for non UBSan build, 51# as UBSan runtime may change any time and thus we could have different number 52# of calls to UBSan runtime even in ASan build. With that, we use the threshold 53# value of 200 that would detect unnecessary UBSan instrumentation in the vast 54# majority of targets, except of a handful very small ones, which would not be 55# a big concern either way as the overhead for them would not be significant. 56UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=200 57 58# ASan builds on i386 generally have about 250 UBSan runtime calls. 59if [[ $ARCHITECTURE == 'i386' ]] 60then 61 UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=280 62fi 63 64 65# Verify that the given fuzz target is correctly built to run with a particular 66# engine. 67function check_engine { 68 local FUZZER=$1 69 local FUZZER_NAME=$(basename $FUZZER) 70 local FUZZER_OUTPUT="/tmp/$FUZZER_NAME.output" 71 local CHECK_FAILED=0 72 73 if [[ "$FUZZING_ENGINE" == libfuzzer ]]; then 74 # Store fuzz target's output into a temp file to be used for further checks. 75 $FUZZER -seed=1337 -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT 76 CHECK_FAILED=$(egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c $FUZZER_OUTPUT) 77 if (( $CHECK_FAILED > 0 )); then 78 echo "BAD BUILD: $FUZZER does not seem to have coverage instrumentation." 79 cat $FUZZER_OUTPUT 80 # Bail out as the further check does not make any sense, there are 0 PCs. 81 return 1 82 fi 83 84 local NUMBER_OF_EDGES=$(grep -Po "INFO: Loaded [[:digit:]]+ module.*\(.*(counters|guards)\):[[:space:]]+\K[[:digit:]]+" $FUZZER_OUTPUT) 85 86 # If a fuzz target fails to start, grep won't find anything, so bail out early to let check_startup_crash deal with it. 87 [[ -z "$NUMBER_OF_EDGES" ]] && return 88 89 if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then 90 echo "BAD BUILD: $FUZZER seems to have only partial coverage instrumentation." 91 fi 92 elif [[ "$FUZZING_ENGINE" == afl ]]; then 93 AFL_FORKSRV_INIT_TMOUT=30000 AFL_NO_UI=1 SKIP_SEED_CORPUS=1 timeout --preserve-status -s INT 35s run_fuzzer $FUZZER_NAME &>$FUZZER_OUTPUT 94 CHECK_PASSED=$(egrep "All set and ready to roll" -c $FUZZER_OUTPUT) 95 if (( $CHECK_PASSED == 0 )); then 96 echo "BAD BUILD: fuzzing $FUZZER with afl-fuzz failed." 97 cat $FUZZER_OUTPUT 98 return 1 99 fi 100 elif [[ "$FUZZING_ENGINE" == honggfuzz ]]; then 101 SKIP_SEED_CORPUS=1 timeout --preserve-status -s INT 20s run_fuzzer $FUZZER_NAME &>$FUZZER_OUTPUT 102 CHECK_PASSED=$(egrep "^Sz:[0-9]+ Tm:[0-9]+" -c $FUZZER_OUTPUT) 103 if (( $CHECK_PASSED == 0 )); then 104 echo "BAD BUILD: fuzzing $FUZZER with honggfuzz failed." 105 cat $FUZZER_OUTPUT 106 return 1 107 fi 108 elif [[ "$FUZZING_ENGINE" == dataflow ]]; then 109 $FUZZER &> $FUZZER_OUTPUT 110 local NUMBER_OF_FUNCTIONS=$(grep -Po "INFO:\s+\K[[:digit:]]+(?=\s+instrumented function.*)" $FUZZER_OUTPUT) 111 [[ -z "$NUMBER_OF_FUNCTIONS" ]] && NUMBER_OF_FUNCTIONS=0 112 if (( $NUMBER_OF_FUNCTIONS < $THRESHOLD_FOR_NUMBER_OF_FUNCTIONS )); then 113 echo "BAD BUILD: $FUZZER does not seem to be properly built in 'dataflow' config." 114 cat $FUZZER_OUTPUT 115 return 1 116 fi 117 return 0 118 fi 119 120 # TODO: add checks for other fuzzing engines if possible. 121 return 0 122} 123 124# Verify that the given fuzz target has been built properly and works. 125function check_startup_crash { 126 local FUZZER=$1 127 local FUZZER_NAME=$(basename $FUZZER) 128 local FUZZER_OUTPUT="/tmp/$FUZZER_NAME.output" 129 local CHECK_PASSED=0 130 131 if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then 132 # Skip seed corpus as there is another explicit check that uses seed corpora. 133 SKIP_SEED_CORPUS=1 run_fuzzer $FUZZER_NAME -seed=1337 -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT 134 CHECK_PASSED=$(egrep "Done $MIN_NUMBER_OF_RUNS runs" -c $FUZZER_OUTPUT) 135 elif [[ "$FUZZING_ENGINE" = afl ]]; then 136 AFL_FORKSRV_INIT_TMOUT=30000 AFL_NO_UI=1 SKIP_SEED_CORPUS=1 timeout --preserve-status -s INT 35s run_fuzzer $FUZZER_NAME &>$FUZZER_OUTPUT 137 if [ $(egrep "target binary (crashed|terminated)" -c $FUZZER_OUTPUT) -eq 0 ]; then 138 CHECK_PASSED=1 139 fi 140 elif [[ "$FUZZING_ENGINE" = dataflow ]]; then 141 # TODO(https://github.com/google/oss-fuzz/issues/1632): add check for 142 # binaries compiled with dataflow engine when the interface becomes stable. 143 CHECK_PASSED=1 144 else 145 # TODO: add checks for another fuzzing engines if possible. 146 CHECK_PASSED=1 147 fi 148 149 if [ "$CHECK_PASSED" -eq "0" ]; then 150 echo "BAD BUILD: $FUZZER seems to have either startup crash or exit:" 151 cat $FUZZER_OUTPUT 152 return 1 153 fi 154 155 return 0 156} 157 158# Mixed sanitizers check for ASan build. 159function check_asan_build { 160 local FUZZER=$1 161 local ASAN_CALLS=$2 162 local DFSAN_CALLS=$3 163 local MSAN_CALLS=$4 164 local UBSAN_CALLS=$5 165 166 # Perform all the checks for more detailed error message. 167 if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then 168 echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan." 169 return 1 170 fi 171 172 if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then 173 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with DFSan." 174 return 1 175 fi 176 177 if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then 178 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan." 179 return 1 180 fi 181 182 if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then 183 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan." 184 return 1 185 fi 186 187 return 0 188} 189 190# Mixed sanitizers check for DFSan build. 191function check_dfsan_build { 192 local FUZZER=$1 193 local ASAN_CALLS=$2 194 local DFSAN_CALLS=$3 195 local MSAN_CALLS=$4 196 local UBSAN_CALLS=$5 197 198 # Perform all the checks for more detailed error message. 199 if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then 200 echo "BAD BUILD: DFSan build of $FUZZER seems to be compiled with ASan." 201 return 1 202 fi 203 204 if (( $DFSAN_CALLS < $DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD )); then 205 echo "BAD BUILD: $FUZZER does not seem to be compiled with DFSan." 206 return 1 207 fi 208 209 if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then 210 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan." 211 return 1 212 fi 213 214 if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then 215 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan." 216 return 1 217 fi 218 219 return 0 220} 221 222 223# Mixed sanitizers check for MSan build. 224function check_msan_build { 225 local FUZZER=$1 226 local ASAN_CALLS=$2 227 local DFSAN_CALLS=$3 228 local MSAN_CALLS=$4 229 local UBSAN_CALLS=$5 230 231 # Perform all the checks for more detailed error message. 232 if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then 233 echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan." 234 return 1 235 fi 236 237 if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then 238 echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with DFSan." 239 return 1 240 fi 241 242 if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then 243 echo "BAD BUILD: $FUZZER does not seem to be compiled with MSan." 244 return 1 245 fi 246 247 if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then 248 echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with UBSan." 249 return 1 250 fi 251 252 return 0 253} 254 255# Mixed sanitizers check for UBSan build. 256function check_ubsan_build { 257 local FUZZER=$1 258 local ASAN_CALLS=$2 259 local DFSAN_CALLS=$3 260 local MSAN_CALLS=$4 261 local UBSAN_CALLS=$5 262 263 if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then 264 # Ignore UBSan checks for fuzzing engines other than libFuzzer because: 265 # A) we (probably) are not going to use those with UBSan 266 # B) such builds show indistinguishable number of calls to UBSan 267 return 0 268 fi 269 270 # Perform all the checks for more detailed error message. 271 if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then 272 echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with ASan." 273 return 1 274 fi 275 276 if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then 277 echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with DFSan." 278 return 1 279 fi 280 281 if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then 282 echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan." 283 return 1 284 fi 285 286 if (( $UBSAN_CALLS < $UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD )); then 287 echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan." 288 return 1 289 fi 290} 291 292# Verify that the given fuzz target is compiled with correct sanitizer. 293function check_mixed_sanitizers { 294 local FUZZER=$1 295 local result=0 296 local CALL_INSN= 297 298 if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then 299 # Sanitizer runtime is linked into the Jazzer driver, so this check does not 300 # apply. 301 return 0 302 fi 303 304 if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then 305 # Sanitizer runtime is loaded via LD_PRELOAD, so this check does not apply. 306 return 0 307 fi 308 309 if [[ $ARCHITECTURE == 'i386' ]] 310 then 311 CALL_INSN="call\s+[0-9a-f]+\s+<" 312 else 313 case $(uname -m) in 314 x86_64) 315 CALL_INSN="callq?\s+[0-9a-f]+\s+<" 316 ;; 317 aarch64) 318 CALL_INSN="bl\s+[0-9a-f]+\s+<" 319 ;; 320 *) 321 echo "Error: unsupported machine hardware $(uname -m)" 322 exit 1 323 ;; 324 esac 325 fi 326 local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__asan" -c) 327 local DFSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__dfsan" -c) 328 local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c) 329 local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c) 330 331 332 if [[ "$SANITIZER" = address ]]; then 333 check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 334 result=$? 335 elif [[ "$SANITIZER" = dataflow ]]; then 336 check_dfsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 337 result=$? 338 elif [[ "$SANITIZER" = memory ]]; then 339 check_msan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 340 result=$? 341 elif [[ "$SANITIZER" = undefined ]]; then 342 check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 343 result=$? 344 elif [[ "$SANITIZER" = thread ]]; then 345 # TODO(metzman): Implement this. 346 result=0 347 fi 348 349 return $result 350} 351 352# Verify that the given fuzz target doesn't crash on the seed corpus. 353function check_seed_corpus { 354 local FUZZER=$1 355 local FUZZER_NAME="$(basename $FUZZER)" 356 local FUZZER_OUTPUT="/tmp/$FUZZER_NAME.output" 357 358 if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then 359 return 0 360 fi 361 362 # Set up common fuzzing arguments, otherwise "run_fuzzer" errors out. 363 if [ -z "$FUZZER_ARGS" ]; then 364 export FUZZER_ARGS="-rss_limit_mb=2560 -timeout=25" 365 fi 366 367 bash -c "run_fuzzer $FUZZER_NAME -runs=0" &> $FUZZER_OUTPUT 368 369 # Don't output anything if fuzz target hasn't crashed. 370 if [ $? -ne 0 ]; then 371 echo "BAD BUILD: $FUZZER has a crashing input in its seed corpus:" 372 cat $FUZZER_OUTPUT 373 return 1 374 fi 375 376 return 0 377} 378 379function check_architecture { 380 local FUZZER=$1 381 local FUZZER_NAME=$(basename $FUZZER) 382 383 if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then 384 # The native dependencies of a JVM project are not packaged, but loaded 385 # dynamically at runtime and thus cannot be checked here. 386 return 0; 387 fi 388 389 if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then 390 FUZZER=${FUZZER}.pkg 391 fi 392 393 FILE_OUTPUT=$(file $FUZZER) 394 if [[ $ARCHITECTURE == "x86_64" ]] 395 then 396 echo $FILE_OUTPUT | grep "x86-64" > /dev/null 397 elif [[ $ARCHITECTURE == "i386" ]] 398 then 399 echo $FILE_OUTPUT | grep "80386" > /dev/null 400 else 401 echo "UNSUPPORTED ARCHITECTURE" 402 return 1 403 fi 404 result=$? 405 if [[ $result != 0 ]] 406 then 407 echo "BAD BUILD $FUZZER is not built for architecture: $ARCHITECTURE" 408 echo "file command output: $FILE_OUTPUT" 409 echo "check_mixed_sanitizers test will fail." 410 fi 411 return $result 412} 413 414function main { 415 local FUZZER=$1 416 local checks_failed=0 417 local result=0 418 419 export RUN_FUZZER_MODE="batch" 420 check_engine $FUZZER 421 result=$? 422 checks_failed=$(( $checks_failed + $result )) 423 424 check_architecture $FUZZER 425 result=$? 426 checks_failed=$(( $checks_failed + $result )) 427 428 check_mixed_sanitizers $FUZZER 429 result=$? 430 checks_failed=$(( $checks_failed + $result )) 431 432 check_startup_crash $FUZZER 433 result=$? 434 checks_failed=$(( $checks_failed + $result )) 435 436 # TODO: re-enable after introducing bug auto-filing for bad builds. 437 # check_seed_corpus $FUZZER 438 return $checks_failed 439} 440 441 442if [ $# -ne 1 ]; then 443 echo "Usage: $0 <fuzz_target_binary>" 444 exit 1 445fi 446 447# Fuzz target path. 448FUZZER=$1 449 450main $FUZZER 451exit $? 452