Lines Matching +full:oss +full:- +full:fuzz +full:- +full:project +full:- +full:name
8 # http://www.apache.org/licenses/LICENSE-2.0
17 """Helper script for OSS-Fuzz users. Can do common tasks like building
38 BASE_RUNNER_IMAGE = 'gcr.io/oss-fuzz-base/base-runner'
42 'gcr.io/oss-fuzz-base/base-image',
43 'gcr.io/oss-fuzz-base/base-clang',
44 'gcr.io/oss-fuzz-base/base-builder',
46 'gcr.io/oss-fuzz-base/base-runner-debug',
48 'go': ['gcr.io/oss-fuzz-base/base-builder-go'],
49 'jvm': ['gcr.io/oss-fuzz-base/base-builder-jvm'],
50 'python': ['gcr.io/oss-fuzz-base/base-builder-python'],
51 'rust': ['gcr.io/oss-fuzz-base/base-builder-rust'],
52 'swift': ['gcr.io/oss-fuzz-base/base-builder-swift'],
55 VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
59 'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
62 'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
73 raw_input = input # pylint: disable=invalid-name
75 # pylint: disable=too-many-lines
78 class Project: class
79 """Class representing a project that is in OSS-Fuzz or an external project
90 self.name = os.path.basename(self.path)
94 self.name = project_name_or_path
95 self.path = os.path.join(OSS_FUZZ_DIR, 'projects', self.name)
100 """Returns path to the project Dockerfile."""
105 """Returns project language."""
110 project_yaml_path = os.path.join(self.path, 'project.yaml')
118 logging.warning('Language not specified in project.yaml.')
123 """Returns the out dir for the project. Creates it if needed."""
124 return _get_out_dir(self.name)
128 """Returns the out dir for the project. Creates it if needed."""
129 return _get_project_build_subdir(self.name, 'work')
133 """Returns the out dir for the project. Creates it if needed."""
134 return _get_project_build_subdir(self.name, 'corpus')
137 def main(): # pylint: disable=too-many-branches,too-many-return-statements
199 project = getattr(parsed_args, 'project', None)
200 if not project:
206 parsed_args.project = Project(parsed_args.project, is_external)
212 '--external',
213 help='Is project external?',
219 def get_parser(): # pylint: disable=too-many-statements
221 parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
225 'generate', help='Generate files for new project.')
226 generate_parser.add_argument('project')
228 '--language',
231 help='Project language.')
236 build_image_parser.add_argument('project')
237 build_image_parser.add_argument('--pull',
240 build_image_parser.add_argument('--cache',
244 build_image_parser.add_argument('--no-pull',
250 'build_fuzzers', help='Build fuzzers for a project.')
256 build_fuzzers_parser.add_argument('project')
260 build_fuzzers_parser.add_argument('--mount_path',
264 build_fuzzers_parser.add_argument('--clean',
268 build_fuzzers_parser.add_argument('--no-clean',
281 check_build_parser.add_argument('project',
282 help='name of the project or path (external)')
284 help='name of the fuzzer',
295 '--corpus-dir', help='directory to store corpus for the fuzz target')
296 run_fuzzer_parser.add_argument('project',
297 help='name of the project or path (external)')
298 run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
304 'coverage', help='Generate code coverage report for the project.')
305 coverage_parser.add_argument('--no-corpus-download',
308 'OSS-Fuzz; use corpus located in '
309 'build/corpus/<project>/<fuzz_target>/')
310 coverage_parser.add_argument('--port',
314 coverage_parser.add_argument('--fuzz-target',
315 help='specify name of a fuzz '
318 coverage_parser.add_argument('--corpus-dir',
320 ' to be used (requires --fuzz-target argument)')
321 coverage_parser.add_argument('project',
322 help='name of the project or path (external)')
325 'pass to llvm-cov utility.',
330 'download_corpora', help='Download all corpora for a project.')
331 download_corpora_parser.add_argument('--fuzz-target',
332 help='specify name of a fuzz target')
334 'project', help='name of the project or path (external)')
338 reproduce_parser.add_argument('--valgrind',
341 reproduce_parser.add_argument('project',
342 help='name of the project or path (external)')
343 reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
353 shell_parser.add_argument('project',
354 help='name of the project or path (external)')
369 """Checks if the image name is a base image."""
370 return os.path.exists(os.path.join('infra', 'base-images', image_name))
373 def check_project_exists(project): argument
374 """Checks if a project exists."""
375 if os.path.exists(project.path):
378 if project.is_external:
379 descriptive_project_name = project.path
381 descriptive_project_name = project.name
387 def _check_fuzzer_exists(project, fuzzer_name): argument
389 command = ['docker', 'run', '--rm']
390 command.extend(['-v', '%s:/out' % project.out])
393 command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])
415 def _get_project_build_subdir(project, subdir_name): argument
416 """Creates the |subdir_name| subdirectory of the |project| subdirectory in
418 directory = os.path.join(BUILD_DIR, subdir_name, project)
425 def _get_out_dir(project=''): argument
426 """Creates and returns path to /out directory for the given project (if
428 return _get_project_build_subdir(project, 'out')
435 parser.add_argument('--architecture',
444 parser.add_argument('--engine',
454 '--sanitizer',
462 parser.add_argument('-e',
467 def build_image_impl(project, cache=True, pull=False): argument
469 image_name = project.name
472 image_project = 'oss-fuzz-base'
473 docker_build_dir = os.path.join(OSS_FUZZ_DIR, 'infra', 'base-images',
477 if not check_project_exists(project):
479 dockerfile_path = project.dockerfile_path
480 docker_build_dir = project.path
481 image_project = 'oss-fuzz'
483 if pull and not pull_images(project.language):
488 build_args.append('--no-cache')
491 '-t',
492 'gcr.io/%s/%s' % (image_project, image_name), '--file', dockerfile_path
500 return sum([['-e', v] for v in env_list], [])
519 def _workdir_from_dockerfile(project): argument
520 """Parses WORKDIR from the Dockerfile for the given project."""
521 with open(project.dockerfile_path) as file_handle:
524 return workdir_from_lines(lines, default=os.path.join('/src', project.name))
529 command = ['docker', 'run', '--rm', '--privileged']
533 command.append('-i')
582 logging.error('Incompatible arguments --pull and --no-pull.')
599 if build_image_impl(args.project, cache=args.cache, pull=pull):
605 def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
606 project, argument
615 if not build_image_impl(project):
621 # Clean old and possibly conflicting artifacts in project's out directory.
623 '-v',
624 '%s:/out' % project.out, '-t',
625 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /out/*'
629 '-v',
630 '%s:/work' % project.work, '-t',
631 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /work/*'
635 logging.info('Keeping existing build artifacts as-is (if any).')
644 if project.language:
645 env.append('FUZZING_LANGUAGE=' + project.language)
650 command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env)
652 workdir = _workdir_from_dockerfile(project)
655 '-v',
664 '-v',
669 '-v',
670 '%s:/out' % project.out, '-v',
671 '%s:/work' % project.work, '-t',
672 'gcr.io/oss-fuzz/%s' % project.name
685 return build_fuzzers_impl(args.project,
704 if not check_project_exists(args.project):
708 not _check_fuzzer_exists(args.project, args.fuzzer_name)):
711 fuzzing_language = args.project.language
714 logging.warning('Language not specified in project.yaml. Defaulting to %s.',
728 '-v', '%s:/out' % args.project.out, '-t', BASE_RUNNER_IMAGE
745 def _get_fuzz_targets(project): argument
746 """Returns names of fuzz targest build in the project's /out directory."""
748 for name in os.listdir(project.out):
749 if name.startswith('afl-'):
751 if name.startswith('jazzer_'):
753 if name == 'llvm-symbolizer':
756 path = os.path.join(project.out, name)
757 # Python and JVM fuzz targets are only executable for the root user, so
760 fuzz_targets.append(name)
765 def _get_latest_corpus(project, fuzz_target, base_corpus_dir): argument
766 """Downloads the latest corpus for the given fuzz target."""
771 if not fuzz_target.startswith(project.name + '_'):
772 fuzz_target = '%s_%s' % (project.name, fuzz_target)
774 corpus_backup_url = CORPUS_BACKUP_URL_FORMAT.format(project_name=project.name,
779 # asking for two-factor authentication.
783 # Some fuzz targets (e.g. new ones) may not have corpus yet, just skip those.
789 latest_backup_url = output.splitlines()[-1]
791 command = ['gsutil', '-q', 'cp', latest_backup_url, archive_path]
794 command = ['unzip', '-q', '-o', archive_path, '-d', corpus_dir]
799 corpus_url = CORPUS_URL_FORMAT.format(project_name=project.name,
801 command = ['gsutil', '-m', '-q', 'rsync', '-R', corpus_url, corpus_dir]
806 """Downloads most recent corpora from GCS for the given project."""
807 if not check_project_exists(args.project):
812 subprocess.check_call(['gsutil', '--version'], stdout=stdout)
821 fuzz_targets = _get_fuzz_targets(args.project)
823 corpus_dir = args.project.corpus
827 _get_latest_corpus(args.project, fuzz_target, corpus_dir)
829 except Exception as error: # pylint:disable=broad-except
834 logging.info('Downloading corpora for %s project to %s.', args.project.name,
844 '--corpus-dir requires specifying a particular fuzz target using '
845 '--fuzz-target')
848 if not check_project_exists(args.project):
851 if args.project.language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT:
853 'Project is written in %s, coverage for it is not supported yet.',
854 args.project.language)
858 not args.project.is_external):
864 'FUZZING_LANGUAGE=%s' % args.project.language,
865 'PROJECT=%s' % args.project.name,
875 '-p',
881 logging.error('The path provided in --corpus-dir argument does not '
885 run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)])
887 run_args.extend(['-v', '%s:/corpus' % args.project.corpus])
890 '-v',
891 '%s:/out' % args.project.out,
892 '-t',
911 if not check_project_exists(args.project):
914 if not _check_fuzzer_exists(args.project, args.fuzzer_name):
930 logging.error('The path provided in --corpus-dir argument does not exist')
934 '-v',
940 '-v',
941 '%s:/out' % args.project.out,
942 '-t',
952 """Reproduces a specific test case from a specific project."""
953 return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e,
957 def reproduce_impl( # pylint: disable=too-many-arguments
958 project, argument
967 if not check_project_exists(project):
970 if not _check_fuzzer_exists(project, fuzzer_name):
975 image_name = 'base-runner'
978 debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full'
981 image_name = 'base-runner-debug'
988 '-v',
989 '%s:/out' % project.out,
990 '-v',
992 '-t',
993 'gcr.io/oss-fuzz-base/%s' % image_name,
996 '-runs=100',
1003 """Validates |project_name| is a valid OSS-Fuzz project name."""
1006 'Project name needs to be less than or equal to %d characters.',
1011 logging.info('Invalid project name: %s.', project_name)
1019 logging.error('Invalid project language %s.', language)
1027 Suitable for OSS-Fuzz and external projects."""
1051 """Generates empty project files."""
1052 return _generate_impl(args.project, args.language)
1063 return 'base-builder'
1064 return 'base-builder-{language}'.format(language=language)
1067 def _generate_impl(project, language): argument
1069 if project.is_external:
1070 # External project.
1073 # Internal project.
1074 if not _validate_project_name(project.name):
1081 directory = project.build_integration_path
1088 'project_name': project.name,
1100 if not build_image_impl(args.project):
1109 if args.project.name != 'base-runner-debug':
1110 env.append('FUZZING_LANGUAGE=' + args.project.language)
1115 if is_base_image(args.project.name):
1116 image_project = 'oss-fuzz-base'
1119 image_project = 'oss-fuzz'
1120 out_dir = args.project.out
1125 '-v',
1130 '-v',
1131 '%s:/out' % out_dir, '-v',
1132 '%s:/work' % args.project.work, '-t',
1133 'gcr.io/%s/%s' % (image_project, args.project.name), '/bin/bash'