1#!/usr/bin/env python 2# Copyright 2016 Google Inc. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17"""Helper script for OSS-Fuzz users. Can do common tasks like building 18projects/fuzzers, running them etc.""" 19 20from __future__ import print_function 21from multiprocessing.dummy import Pool as ThreadPool 22import argparse 23import datetime 24import errno 25import os 26import pipes 27import re 28import subprocess 29import sys 30import templates 31 32OSS_FUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 33BUILD_DIR = os.path.join(OSS_FUZZ_DIR, 'build') 34 35BASE_IMAGES = [ 36 'gcr.io/oss-fuzz-base/base-image', 37 'gcr.io/oss-fuzz-base/base-clang', 38 'gcr.io/oss-fuzz-base/base-builder', 39 'gcr.io/oss-fuzz-base/base-runner', 40 'gcr.io/oss-fuzz-base/base-runner-debug', 41 'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 42 'gcr.io/oss-fuzz-base/msan-libs-builder', 43] 44 45VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$') 46MAX_PROJECT_NAME_LENGTH = 26 47 48if sys.version_info[0] >= 3: 49 raw_input = input # pylint: disable=invalid-name 50 51CORPUS_URL_FORMAT = ( 52 'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/' 53 '{fuzz_target}/') 54CORPUS_BACKUP_URL_FORMAT = ( 55 'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/' 56 'libFuzzer/{fuzz_target}/') 57 58PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)') 59 60# Languages from project.yaml that have code coverage support. 61LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust'] 62 63# pylint: disable=too-many-lines 64 65 66def main(): # pylint: disable=too-many-branches,too-many-return-statements 67 """Get subcommand from program arguments and do it.""" 68 os.chdir(OSS_FUZZ_DIR) 69 if not os.path.exists(BUILD_DIR): 70 os.mkdir(BUILD_DIR) 71 72 args = parse_args() 73 74 # We have different default values for `sanitizer` depending on the `engine`. 75 # Some commands do not have `sanitizer` argument, so `hasattr` is necessary. 76 if hasattr(args, 'sanitizer') and not args.sanitizer: 77 if args.engine == 'dataflow': 78 args.sanitizer = 'dataflow' 79 else: 80 args.sanitizer = 'address' 81 82 if args.command == 'generate': 83 return generate(args) 84 if args.command == 'build_image': 85 return build_image(args) 86 if args.command == 'build_fuzzers': 87 return build_fuzzers(args) 88 if args.command == 'check_build': 89 return check_build(args) 90 if args.command == 'download_corpora': 91 return download_corpora(args) 92 if args.command == 'run_fuzzer': 93 return run_fuzzer(args) 94 if args.command == 'coverage': 95 return coverage(args) 96 if args.command == 'reproduce': 97 return reproduce(args) 98 if args.command == 'shell': 99 return shell(args) 100 if args.command == 'pull_images': 101 return pull_images(args) 102 103 return 0 104 105 106def parse_args(args=None): 107 """Parses args using argparser and returns parsed args.""" 108 # Use default argument None for args so that in production, argparse does its 109 # normal behavior, but unittesting is easier. 110 parser = get_parser() 111 return parser.parse_args(args) 112 113 114def get_parser(): # pylint: disable=too-many-statements 115 """Returns an argparse parser.""" 116 parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers') 117 subparsers = parser.add_subparsers(dest='command') 118 119 generate_parser = subparsers.add_parser( 120 'generate', help='Generate files for new project.') 121 generate_parser.add_argument('project_name') 122 123 build_image_parser = subparsers.add_parser('build_image', 124 help='Build an image.') 125 build_image_parser.add_argument('project_name') 126 build_image_parser.add_argument('--pull', 127 action='store_true', 128 help='Pull latest base image.') 129 build_image_parser.add_argument('--no-pull', 130 action='store_true', 131 help='Do not pull latest base image.') 132 133 build_fuzzers_parser = subparsers.add_parser( 134 'build_fuzzers', help='Build fuzzers for a project.') 135 _add_architecture_args(build_fuzzers_parser) 136 _add_engine_args(build_fuzzers_parser) 137 _add_sanitizer_args(build_fuzzers_parser) 138 _add_environment_args(build_fuzzers_parser) 139 build_fuzzers_parser.add_argument('project_name') 140 build_fuzzers_parser.add_argument('source_path', 141 help='path of local source', 142 nargs='?') 143 build_fuzzers_parser.add_argument('--clean', 144 dest='clean', 145 action='store_true', 146 help='clean existing artifacts.') 147 build_fuzzers_parser.add_argument('--no-clean', 148 dest='clean', 149 action='store_false', 150 help='do not clean existing artifacts ' 151 '(default).') 152 build_fuzzers_parser.set_defaults(clean=False) 153 154 check_build_parser = subparsers.add_parser( 155 'check_build', help='Checks that fuzzers execute without errors.') 156 _add_architecture_args(check_build_parser) 157 _add_engine_args( 158 check_build_parser, 159 choices=['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none']) 160 _add_sanitizer_args( 161 check_build_parser, 162 choices=['address', 'memory', 'undefined', 'dataflow', 'thread']) 163 _add_environment_args(check_build_parser) 164 check_build_parser.add_argument('project_name', help='name of the project') 165 check_build_parser.add_argument('fuzzer_name', 166 help='name of the fuzzer', 167 nargs='?') 168 169 run_fuzzer_parser = subparsers.add_parser( 170 'run_fuzzer', help='Run a fuzzer in the emulated fuzzing environment.') 171 _add_engine_args(run_fuzzer_parser) 172 _add_sanitizer_args(run_fuzzer_parser) 173 _add_environment_args(run_fuzzer_parser) 174 run_fuzzer_parser.add_argument( 175 '--corpus-dir', help='directory to store corpus for the fuzz target') 176 run_fuzzer_parser.add_argument('project_name', help='name of the project') 177 run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer') 178 run_fuzzer_parser.add_argument('fuzzer_args', 179 help='arguments to pass to the fuzzer', 180 nargs=argparse.REMAINDER) 181 182 coverage_parser = subparsers.add_parser( 183 'coverage', help='Generate code coverage report for the project.') 184 coverage_parser.add_argument('--no-corpus-download', 185 action='store_true', 186 help='do not download corpus backup from ' 187 'OSS-Fuzz; use corpus located in ' 188 'build/corpus/<project>/<fuzz_target>/') 189 coverage_parser.add_argument('--port', 190 default='8008', 191 help='specify port for' 192 ' a local HTTP server rendering coverage report') 193 coverage_parser.add_argument('--fuzz-target', 194 help='specify name of a fuzz ' 195 'target to be run for generating coverage ' 196 'report') 197 coverage_parser.add_argument('--corpus-dir', 198 help='specify location of corpus' 199 ' to be used (requires --fuzz-target argument)') 200 coverage_parser.add_argument('project_name', help='name of the project') 201 coverage_parser.add_argument('extra_args', 202 help='additional arguments to ' 203 'pass to llvm-cov utility.', 204 nargs='*') 205 206 download_corpora_parser = subparsers.add_parser( 207 'download_corpora', help='Download all corpora for a project.') 208 download_corpora_parser.add_argument('--fuzz-target', 209 help='specify name of a fuzz target') 210 download_corpora_parser.add_argument('project_name', 211 help='name of the project') 212 213 reproduce_parser = subparsers.add_parser('reproduce', 214 help='Reproduce a crash.') 215 reproduce_parser.add_argument('--valgrind', 216 action='store_true', 217 help='run with valgrind') 218 reproduce_parser.add_argument('project_name', help='name of the project') 219 reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer') 220 reproduce_parser.add_argument('testcase_path', help='path of local testcase') 221 reproduce_parser.add_argument('fuzzer_args', 222 help='arguments to pass to the fuzzer', 223 nargs=argparse.REMAINDER) 224 _add_environment_args(reproduce_parser) 225 226 shell_parser = subparsers.add_parser( 227 'shell', help='Run /bin/bash within the builder container.') 228 shell_parser.add_argument('project_name', help='name of the project') 229 shell_parser.add_argument('source_path', 230 help='path of local source', 231 nargs='?') 232 _add_architecture_args(shell_parser) 233 _add_engine_args(shell_parser) 234 _add_sanitizer_args(shell_parser) 235 _add_environment_args(shell_parser) 236 237 subparsers.add_parser('pull_images', help='Pull base images.') 238 return parser 239 240 241def is_base_image(image_name): 242 """Checks if the image name is a base image.""" 243 return os.path.exists(os.path.join('infra', 'base-images', image_name)) 244 245 246def check_project_exists(project_name): 247 """Checks if a project exists.""" 248 if not os.path.exists(_get_project_dir(project_name)): 249 print(project_name, 'does not exist', file=sys.stderr) 250 return False 251 252 return True 253 254 255def _check_fuzzer_exists(project_name, fuzzer_name): 256 """Checks if a fuzzer exists.""" 257 command = ['docker', 'run', '--rm'] 258 command.extend(['-v', '%s:/out' % _get_output_dir(project_name)]) 259 command.append('ubuntu:16.04') 260 261 command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name]) 262 263 try: 264 subprocess.check_call(command) 265 except subprocess.CalledProcessError: 266 print(fuzzer_name, 267 'does not seem to exist. Please run build_fuzzers first.', 268 file=sys.stderr) 269 return False 270 271 return True 272 273 274def _get_absolute_path(path): 275 """Returns absolute path with user expansion.""" 276 return os.path.abspath(os.path.expanduser(path)) 277 278 279def _get_command_string(command): 280 """Returns a shell escaped command string.""" 281 return ' '.join(pipes.quote(part) for part in command) 282 283 284def _get_project_dir(project_name): 285 """Returns path to the project.""" 286 return os.path.join(OSS_FUZZ_DIR, 'projects', project_name) 287 288 289def get_dockerfile_path(project_name): 290 """Returns path to the project Dockerfile.""" 291 return os.path.join(_get_project_dir(project_name), 'Dockerfile') 292 293 294def _get_corpus_dir(project_name=''): 295 """Creates and returns path to /corpus directory for the given project (if 296 specified).""" 297 directory = os.path.join(BUILD_DIR, 'corpus', project_name) 298 if not os.path.exists(directory): 299 os.makedirs(directory) 300 301 return directory 302 303 304def _get_output_dir(project_name=''): 305 """Creates and returns path to /out directory for the given project (if 306 specified).""" 307 directory = os.path.join(BUILD_DIR, 'out', project_name) 308 if not os.path.exists(directory): 309 os.makedirs(directory) 310 311 return directory 312 313 314def _get_work_dir(project_name=''): 315 """Creates and returns path to /work directory for the given project (if 316 specified).""" 317 directory = os.path.join(BUILD_DIR, 'work', project_name) 318 if not os.path.exists(directory): 319 os.makedirs(directory) 320 321 return directory 322 323 324def _get_project_language(project_name): 325 """Returns project language.""" 326 project_yaml_path = os.path.join(OSS_FUZZ_DIR, 'projects', project_name, 327 'project.yaml') 328 with open(project_yaml_path) as file_handle: 329 content = file_handle.read() 330 for line in content.splitlines(): 331 match = PROJECT_LANGUAGE_REGEX.match(line) 332 if match: 333 return match.group(1) 334 335 return None 336 337 338def _add_architecture_args(parser, choices=('x86_64', 'i386')): 339 """Add common architecture args.""" 340 parser.add_argument('--architecture', default='x86_64', choices=choices) 341 342 343def _add_engine_args(parser, 344 choices=('libfuzzer', 'afl', 'honggfuzz', 'dataflow', 345 'none')): 346 """Add common engine args.""" 347 parser.add_argument('--engine', default='libfuzzer', choices=choices) 348 349 350def _add_sanitizer_args(parser, 351 choices=('address', 'memory', 'undefined', 'coverage', 352 'dataflow', 'thread')): 353 """Add common sanitizer args.""" 354 parser.add_argument( 355 '--sanitizer', 356 default=None, 357 choices=choices, 358 help='the default is "address"; "dataflow" for "dataflow" engine') 359 360 361def _add_environment_args(parser): 362 """Add common environment args.""" 363 parser.add_argument('-e', 364 action='append', 365 help="set environment variable e.g. VAR=value") 366 367 368def build_image_impl(image_name, no_cache=False, pull=False): 369 """Build image.""" 370 371 proj_is_base_image = is_base_image(image_name) 372 if proj_is_base_image: 373 image_project = 'oss-fuzz-base' 374 dockerfile_dir = os.path.join('infra', 'base-images', image_name) 375 else: 376 image_project = 'oss-fuzz' 377 if not check_project_exists(image_name): 378 return False 379 380 dockerfile_dir = os.path.join('projects', image_name) 381 382 build_args = [] 383 if no_cache: 384 build_args.append('--no-cache') 385 386 build_args += [ 387 '-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir 388 ] 389 390 return docker_build(build_args, pull=pull) 391 392 393def _env_to_docker_args(env_list): 394 """Turn envirnoment variable list into docker arguments.""" 395 return sum([['-e', v] for v in env_list], []) 396 397 398WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') 399 400 401def workdir_from_lines(lines, default='/src'): 402 """Get the WORKDIR from the given lines.""" 403 for line in reversed(lines): # reversed to get last WORKDIR. 404 match = re.match(WORKDIR_REGEX, line) 405 if match: 406 workdir = match.group(1) 407 workdir = workdir.replace('$SRC', '/src') 408 409 if not os.path.isabs(workdir): 410 workdir = os.path.join('/src', workdir) 411 412 return os.path.normpath(workdir) 413 414 return default 415 416 417def _workdir_from_dockerfile(project_name): 418 """Parse WORKDIR from the Dockerfile for the given project.""" 419 dockerfile_path = get_dockerfile_path(project_name) 420 421 with open(dockerfile_path) as file_handle: 422 lines = file_handle.readlines() 423 424 return workdir_from_lines(lines, default=os.path.join('/src', project_name)) 425 426 427def docker_run(run_args, print_output=True): 428 """Call `docker run`.""" 429 command = ['docker', 'run', '--rm', '--privileged'] 430 431 # Support environments with a TTY. 432 if sys.stdin.isatty(): 433 command.append('-i') 434 435 command.extend(run_args) 436 437 print('Running:', _get_command_string(command)) 438 stdout = None 439 if not print_output: 440 stdout = open(os.devnull, 'w') 441 442 try: 443 subprocess.check_call(command, stdout=stdout, stderr=subprocess.STDOUT) 444 except subprocess.CalledProcessError as error: 445 return error.returncode 446 447 return 0 448 449 450def docker_build(build_args, pull=False): 451 """Call `docker build`.""" 452 command = ['docker', 'build'] 453 if pull: 454 command.append('--pull') 455 456 command.extend(build_args) 457 print('Running:', _get_command_string(command)) 458 459 try: 460 subprocess.check_call(command) 461 except subprocess.CalledProcessError: 462 print('docker build failed.', file=sys.stderr) 463 return False 464 465 return True 466 467 468def docker_pull(image): 469 """Call `docker pull`.""" 470 command = ['docker', 'pull', image] 471 print('Running:', _get_command_string(command)) 472 473 try: 474 subprocess.check_call(command) 475 except subprocess.CalledProcessError: 476 print('docker pull failed.', file=sys.stderr) 477 return False 478 479 return True 480 481 482def build_image(args): 483 """Build docker image.""" 484 if args.pull and args.no_pull: 485 print('Incompatible arguments --pull and --no-pull.') 486 return 1 487 488 if args.pull: 489 pull = True 490 elif args.no_pull: 491 pull = False 492 else: 493 y_or_n = raw_input('Pull latest base images (compiler/runtime)? (y/N): ') 494 pull = y_or_n.lower() == 'y' 495 496 if pull: 497 print('Pulling latest base images...') 498 else: 499 print('Using cached base images...') 500 501 # If build_image is called explicitly, don't use cache. 502 if build_image_impl(args.project_name, no_cache=True, pull=pull): 503 return 0 504 505 return 1 506 507 508def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches 509 project_name, 510 clean, 511 engine, 512 sanitizer, 513 architecture, 514 env_to_add, 515 source_path, 516 no_cache=False, 517 mount_location=None): 518 """Build fuzzers.""" 519 if not build_image_impl(project_name, no_cache=no_cache): 520 return 1 521 522 project_out_dir = _get_output_dir(project_name) 523 project_work_dir = _get_work_dir(project_name) 524 project_language = _get_project_language(project_name) 525 if not project_language: 526 print('WARNING: language not specified in project.yaml. Build may fail.') 527 528 if clean: 529 print('Cleaning existing build artifacts.') 530 531 # Clean old and possibly conflicting artifacts in project's out directory. 532 docker_run([ 533 '-v', 534 '%s:/out' % project_out_dir, '-t', 535 'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /out/*' 536 ]) 537 538 docker_run([ 539 '-v', 540 '%s:/work' % project_work_dir, '-t', 541 'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /work/*' 542 ]) 543 544 else: 545 print('Keeping existing build artifacts as-is (if any).') 546 env = [ 547 'FUZZING_ENGINE=' + engine, 548 'SANITIZER=' + sanitizer, 549 'ARCHITECTURE=' + architecture, 550 ] 551 552 if project_language: 553 env.append('FUZZING_LANGUAGE=' + project_language) 554 555 if env_to_add: 556 env += env_to_add 557 558 # Copy instrumented libraries. 559 if sanitizer == 'memory': 560 docker_run([ 561 '-v', 562 '%s:/work' % project_work_dir, 'gcr.io/oss-fuzz-base/msan-libs-builder', 563 'bash', '-c', 'cp -r /msan /work' 564 ]) 565 env.append('MSAN_LIBS_PATH=' + '/work/msan') 566 567 command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env) 568 if source_path: 569 workdir = _workdir_from_dockerfile(project_name) 570 if mount_location: 571 command += [ 572 '-v', 573 '%s:%s' % (_get_absolute_path(source_path), mount_location), 574 ] 575 else: 576 if workdir == '/src': 577 print('Cannot use local checkout with "WORKDIR: /src".', 578 file=sys.stderr) 579 return 1 580 581 command += [ 582 '-v', 583 '%s:%s' % (_get_absolute_path(source_path), workdir), 584 ] 585 586 command += [ 587 '-v', 588 '%s:/out' % project_out_dir, '-v', 589 '%s:/work' % project_work_dir, '-t', 590 'gcr.io/oss-fuzz/%s' % project_name 591 ] 592 593 result_code = docker_run(command) 594 if result_code: 595 print('Building fuzzers failed.', file=sys.stderr) 596 return result_code 597 598 # Patch MSan builds to use instrumented shared libraries. 599 if sanitizer == 'memory': 600 docker_run([ 601 '-v', 602 '%s:/out' % project_out_dir, '-v', 603 '%s:/work' % project_work_dir 604 ] + _env_to_docker_args(env) + [ 605 'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 'patch_build.py', 606 '/out' 607 ]) 608 609 return 0 610 611 612def build_fuzzers(args): 613 """Build fuzzers.""" 614 return build_fuzzers_impl(args.project_name, args.clean, args.engine, 615 args.sanitizer, args.architecture, args.e, 616 args.source_path) 617 618 619def check_build(args): 620 """Checks that fuzzers in the container execute without errors.""" 621 if not check_project_exists(args.project_name): 622 return 1 623 624 if (args.fuzzer_name and 625 not _check_fuzzer_exists(args.project_name, args.fuzzer_name)): 626 return 1 627 628 fuzzing_language = _get_project_language(args.project_name) 629 if fuzzing_language is None: 630 print('WARNING: language not specified in project.yaml. Defaulting to C++.') 631 fuzzing_language = 'c++' 632 633 env = [ 634 'FUZZING_ENGINE=' + args.engine, 635 'SANITIZER=' + args.sanitizer, 636 'ARCHITECTURE=' + args.architecture, 637 'FUZZING_LANGUAGE=' + fuzzing_language, 638 ] 639 if args.e: 640 env += args.e 641 642 run_args = _env_to_docker_args(env) + [ 643 '-v', 644 '%s:/out' % _get_output_dir(args.project_name), '-t', 645 'gcr.io/oss-fuzz-base/base-runner' 646 ] 647 648 if args.fuzzer_name: 649 run_args += ['test_one.py', args.fuzzer_name] 650 else: 651 run_args.append('test_all.py') 652 653 exit_code = docker_run(run_args) 654 if exit_code == 0: 655 print('Check build passed.') 656 else: 657 print('Check build failed.') 658 659 return exit_code 660 661 662def _get_fuzz_targets(project_name): 663 """Return names of fuzz targest build in the project's /out directory.""" 664 fuzz_targets = [] 665 for name in os.listdir(_get_output_dir(project_name)): 666 if name.startswith('afl-'): 667 continue 668 669 path = os.path.join(_get_output_dir(project_name), name) 670 if os.path.isfile(path) and os.access(path, os.X_OK): 671 fuzz_targets.append(name) 672 673 return fuzz_targets 674 675 676def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir): 677 """Download the latest corpus for the given fuzz target.""" 678 corpus_dir = os.path.join(base_corpus_dir, fuzz_target) 679 if not os.path.exists(corpus_dir): 680 os.makedirs(corpus_dir) 681 682 if not fuzz_target.startswith(project_name + '_'): 683 fuzz_target = '%s_%s' % (project_name, fuzz_target) 684 685 corpus_backup_url = CORPUS_BACKUP_URL_FORMAT.format(project_name=project_name, 686 fuzz_target=fuzz_target) 687 command = ['gsutil', 'ls', corpus_backup_url] 688 689 # Don't capture stderr. We want it to print in real time, in case gsutil is 690 # asking for two-factor authentication. 691 corpus_listing = subprocess.Popen(command, stdout=subprocess.PIPE) 692 output, _ = corpus_listing.communicate() 693 694 # Some fuzz targets (e.g. new ones) may not have corpus yet, just skip those. 695 if corpus_listing.returncode: 696 print('WARNING: corpus for {0} not found:\n'.format(fuzz_target), 697 file=sys.stderr) 698 return 699 700 if output: 701 latest_backup_url = output.splitlines()[-1] 702 archive_path = corpus_dir + '.zip' 703 command = ['gsutil', '-q', 'cp', latest_backup_url, archive_path] 704 subprocess.check_call(command) 705 706 command = ['unzip', '-q', '-o', archive_path, '-d', corpus_dir] 707 subprocess.check_call(command) 708 os.remove(archive_path) 709 else: 710 # Sync the working corpus copy if a minimized backup is not available. 711 corpus_url = CORPUS_URL_FORMAT.format(project_name=project_name, 712 fuzz_target=fuzz_target) 713 command = ['gsutil', '-m', '-q', 'rsync', '-R', corpus_url, corpus_dir] 714 subprocess.check_call(command) 715 716 717def download_corpora(args): 718 """Download most recent corpora from GCS for the given project.""" 719 if not check_project_exists(args.project_name): 720 return 1 721 722 try: 723 with open(os.devnull, 'w') as stdout: 724 subprocess.check_call(['gsutil', '--version'], stdout=stdout) 725 except OSError: 726 print( 727 'ERROR: gsutil not found. Please install it from ' 728 'https://cloud.google.com/storage/docs/gsutil_install', 729 file=sys.stderr) 730 return False 731 732 if args.fuzz_target: 733 fuzz_targets = [args.fuzz_target] 734 else: 735 fuzz_targets = _get_fuzz_targets(args.project_name) 736 737 corpus_dir = _get_corpus_dir(args.project_name) 738 if not os.path.exists(corpus_dir): 739 os.makedirs(corpus_dir) 740 741 def _download_for_single_target(fuzz_target): 742 try: 743 _get_latest_corpus(args.project_name, fuzz_target, corpus_dir) 744 return True 745 except Exception as error: # pylint:disable=broad-except 746 print('ERROR: corpus download for %s failed: %s' % 747 (fuzz_target, str(error)), 748 file=sys.stderr) 749 return False 750 751 print('Downloading corpora for %s project to %s' % 752 (args.project_name, corpus_dir)) 753 thread_pool = ThreadPool() 754 return all(thread_pool.map(_download_for_single_target, fuzz_targets)) 755 756 757def coverage(args): 758 """Generate code coverage using clang source based code coverage.""" 759 if args.corpus_dir and not args.fuzz_target: 760 print( 761 'ERROR: --corpus-dir requires specifying a particular fuzz target ' 762 'using --fuzz-target', 763 file=sys.stderr) 764 return 1 765 766 if not check_project_exists(args.project_name): 767 return 1 768 769 project_language = _get_project_language(args.project_name) 770 if project_language not in LANGUAGES_WITH_COVERAGE_SUPPORT: 771 print( 772 'ERROR: Project is written in %s, coverage for it is not supported yet.' 773 % project_language, 774 file=sys.stderr) 775 return 1 776 777 if not args.no_corpus_download and not args.corpus_dir: 778 if not download_corpora(args): 779 return 1 780 781 env = [ 782 'FUZZING_ENGINE=libfuzzer', 783 'FUZZING_LANGUAGE=%s' % project_language, 784 'PROJECT=%s' % args.project_name, 785 'SANITIZER=coverage', 786 'HTTP_PORT=%s' % args.port, 787 'COVERAGE_EXTRA_ARGS=%s' % ' '.join(args.extra_args), 788 ] 789 790 run_args = _env_to_docker_args(env) 791 792 if args.port: 793 run_args.extend([ 794 '-p', 795 '%s:%s' % (args.port, args.port), 796 ]) 797 798 if args.corpus_dir: 799 if not os.path.exists(args.corpus_dir): 800 print('ERROR: the path provided in --corpus-dir argument does not exist', 801 file=sys.stderr) 802 return 1 803 corpus_dir = os.path.realpath(args.corpus_dir) 804 run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)]) 805 else: 806 run_args.extend(['-v', '%s:/corpus' % _get_corpus_dir(args.project_name)]) 807 808 run_args.extend([ 809 '-v', 810 '%s:/out' % _get_output_dir(args.project_name), 811 '-t', 812 'gcr.io/oss-fuzz-base/base-runner', 813 ]) 814 815 run_args.append('coverage') 816 if args.fuzz_target: 817 run_args.append(args.fuzz_target) 818 819 exit_code = docker_run(run_args) 820 if exit_code == 0: 821 print('Successfully generated clang code coverage report.') 822 else: 823 print('Failed to generate clang code coverage report.') 824 825 return exit_code 826 827 828def run_fuzzer(args): 829 """Runs a fuzzer in the container.""" 830 if not check_project_exists(args.project_name): 831 return 1 832 833 if not _check_fuzzer_exists(args.project_name, args.fuzzer_name): 834 return 1 835 836 env = [ 837 'FUZZING_ENGINE=' + args.engine, 838 'SANITIZER=' + args.sanitizer, 839 'RUN_FUZZER_MODE=interactive', 840 ] 841 842 if args.e: 843 env += args.e 844 845 run_args = _env_to_docker_args(env) 846 847 if args.corpus_dir: 848 if not os.path.exists(args.corpus_dir): 849 print('ERROR: the path provided in --corpus-dir argument does not exist', 850 file=sys.stderr) 851 return 1 852 corpus_dir = os.path.realpath(args.corpus_dir) 853 run_args.extend([ 854 '-v', 855 '{corpus_dir}:/tmp/{fuzzer}_corpus'.format(corpus_dir=corpus_dir, 856 fuzzer=args.fuzzer_name) 857 ]) 858 859 run_args.extend([ 860 '-v', 861 '%s:/out' % _get_output_dir(args.project_name), 862 '-t', 863 'gcr.io/oss-fuzz-base/base-runner', 864 'run_fuzzer', 865 args.fuzzer_name, 866 ] + args.fuzzer_args) 867 868 return docker_run(run_args) 869 870 871def reproduce(args): 872 """Reproduce a specific test case from a specific project.""" 873 return reproduce_impl(args.project_name, args.fuzzer_name, args.valgrind, 874 args.e, args.fuzzer_args, args.testcase_path) 875 876 877def reproduce_impl( # pylint: disable=too-many-arguments 878 project_name, 879 fuzzer_name, 880 valgrind, 881 env_to_add, 882 fuzzer_args, 883 testcase_path, 884 runner=docker_run, 885 err_result=1): 886 """Reproduces a testcase in the container.""" 887 if not check_project_exists(project_name): 888 return err_result 889 890 if not _check_fuzzer_exists(project_name, fuzzer_name): 891 return err_result 892 893 debugger = '' 894 env = [] 895 image_name = 'base-runner' 896 897 if valgrind: 898 debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full' 899 900 if debugger: 901 image_name = 'base-runner-debug' 902 env += ['DEBUGGER=' + debugger] 903 904 if env_to_add: 905 env += env_to_add 906 907 run_args = _env_to_docker_args(env) + [ 908 '-v', 909 '%s:/out' % _get_output_dir(project_name), 910 '-v', 911 '%s:/testcase' % _get_absolute_path(testcase_path), 912 '-t', 913 'gcr.io/oss-fuzz-base/%s' % image_name, 914 'reproduce', 915 fuzzer_name, 916 '-runs=100', 917 ] + fuzzer_args 918 919 return runner(run_args) 920 921 922def generate(args): 923 """Generate empty project files.""" 924 if len(args.project_name) > MAX_PROJECT_NAME_LENGTH: 925 print('Project name needs to be less than or equal to %d characters.' % 926 MAX_PROJECT_NAME_LENGTH, 927 file=sys.stderr) 928 return 1 929 930 if not VALID_PROJECT_NAME_REGEX.match(args.project_name): 931 print('Invalid project name.', file=sys.stderr) 932 return 1 933 934 directory = os.path.join('projects', args.project_name) 935 936 try: 937 os.mkdir(directory) 938 except OSError as error: 939 if error.errno != errno.EEXIST: 940 raise 941 print(directory, 'already exists.', file=sys.stderr) 942 return 1 943 944 print('Writing new files to', directory) 945 946 template_args = { 947 'project_name': args.project_name, 948 'year': datetime.datetime.now().year 949 } 950 with open(os.path.join(directory, 'project.yaml'), 'w') as file_handle: 951 file_handle.write(templates.PROJECT_YAML_TEMPLATE % template_args) 952 953 with open(os.path.join(directory, 'Dockerfile'), 'w') as file_handle: 954 file_handle.write(templates.DOCKER_TEMPLATE % template_args) 955 956 build_sh_path = os.path.join(directory, 'build.sh') 957 with open(build_sh_path, 'w') as file_handle: 958 file_handle.write(templates.BUILD_TEMPLATE % template_args) 959 960 os.chmod(build_sh_path, 0o755) 961 return 0 962 963 964def shell(args): 965 """Runs a shell within a docker image.""" 966 if not build_image_impl(args.project_name): 967 return 1 968 969 env = [ 970 'FUZZING_ENGINE=' + args.engine, 971 'SANITIZER=' + args.sanitizer, 972 'ARCHITECTURE=' + args.architecture, 973 ] 974 975 if args.project_name != 'base-runner-debug': 976 env.append('FUZZING_LANGUAGE=' + _get_project_language(args.project_name)) 977 978 if args.e: 979 env += args.e 980 981 if is_base_image(args.project_name): 982 image_project = 'oss-fuzz-base' 983 out_dir = _get_output_dir() 984 else: 985 image_project = 'oss-fuzz' 986 out_dir = _get_output_dir(args.project_name) 987 988 run_args = _env_to_docker_args(env) 989 if args.source_path: 990 run_args.extend([ 991 '-v', 992 '%s:%s' % (_get_absolute_path(args.source_path), '/src'), 993 ]) 994 995 run_args.extend([ 996 '-v', 997 '%s:/out' % out_dir, '-v', 998 '%s:/work' % _get_work_dir(args.project_name), '-t', 999 'gcr.io/%s/%s' % (image_project, args.project_name), '/bin/bash' 1000 ]) 1001 1002 docker_run(run_args) 1003 return 0 1004 1005 1006def pull_images(_): 1007 """Pull base images.""" 1008 for base_image in BASE_IMAGES: 1009 if not docker_pull(base_image): 1010 return 1 1011 1012 return 0 1013 1014 1015if __name__ == '__main__': 1016 sys.exit(main()) 1017