Lines Matching +full:project +full:- +full:src +full:- +full:path
8 # http://www.apache.org/licenses/LICENSE-2.0
17 """Helper script for OSS-Fuzz users. Can do common tasks like building
35 OSS_FUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
36 BUILD_DIR = os.path.join(OSS_FUZZ_DIR, 'build')
38 BASE_RUNNER_IMAGE = 'gcr.io/oss-fuzz-base/base-runner'
42 'gcr.io/oss-fuzz-base/base-image',
43 'gcr.io/oss-fuzz-base/base-clang',
44 'gcr.io/oss-fuzz-base/base-builder',
46 'gcr.io/oss-fuzz-base/base-runner-debug',
48 'go': ['gcr.io/oss-fuzz-base/base-builder-go'],
49 'jvm': ['gcr.io/oss-fuzz-base/base-builder-jvm'],
50 'python': ['gcr.io/oss-fuzz-base/base-builder-python'],
51 'rust': ['gcr.io/oss-fuzz-base/base-builder-rust'],
52 'swift': ['gcr.io/oss-fuzz-base/base-builder-swift'],
55 VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
59 'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
62 'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
73 raw_input = input # pylint: disable=invalid-name
75 # pylint: disable=too-many-lines
78 class Project: class
79 """Class representing a project that is in OSS-Fuzz or an external project
89 self.path = os.path.abspath(project_name_or_path)
90 self.name = os.path.basename(self.path)
91 self.build_integration_path = os.path.join(self.path,
95 self.path = os.path.join(OSS_FUZZ_DIR, 'projects', self.name)
96 self.build_integration_path = self.path
100 """Returns path to the project Dockerfile."""
101 return os.path.join(self.build_integration_path, 'Dockerfile')
105 """Returns project language."""
110 project_yaml_path = os.path.join(self.path, 'project.yaml')
118 logging.warning('Language not specified in project.yaml.')
123 """Returns the out dir for the project. Creates it if needed."""
128 """Returns the out dir for the project. Creates it if needed."""
133 """Returns the out dir for the project. Creates it if needed."""
137 def main(): # pylint: disable=too-many-branches,too-many-return-statements
148 if not os.path.exists(BUILD_DIR):
199 project = getattr(parsed_args, 'project', None)
200 if not project:
206 parsed_args.project = Project(parsed_args.project, is_external)
212 '--external',
213 help='Is project external?',
219 def get_parser(): # pylint: disable=too-many-statements
221 parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
225 'generate', help='Generate files for new project.')
226 generate_parser.add_argument('project')
228 '--language',
231 help='Project language.')
236 build_image_parser.add_argument('project')
237 build_image_parser.add_argument('--pull',
240 build_image_parser.add_argument('--cache',
244 build_image_parser.add_argument('--no-pull',
250 'build_fuzzers', help='Build fuzzers for a project.')
256 build_fuzzers_parser.add_argument('project')
258 help='path of local source',
260 build_fuzzers_parser.add_argument('--mount_path',
262 help='path to mount local source in '
264 build_fuzzers_parser.add_argument('--clean',
268 build_fuzzers_parser.add_argument('--no-clean',
281 check_build_parser.add_argument('project',
282 help='name of the project or path (external)')
295 '--corpus-dir', help='directory to store corpus for the fuzz target')
296 run_fuzzer_parser.add_argument('project',
297 help='name of the project or path (external)')
304 'coverage', help='Generate code coverage report for the project.')
305 coverage_parser.add_argument('--no-corpus-download',
308 'OSS-Fuzz; use corpus located in '
309 'build/corpus/<project>/<fuzz_target>/')
310 coverage_parser.add_argument('--port',
314 coverage_parser.add_argument('--fuzz-target',
318 coverage_parser.add_argument('--corpus-dir',
320 ' to be used (requires --fuzz-target argument)')
321 coverage_parser.add_argument('project',
322 help='name of the project or path (external)')
325 'pass to llvm-cov utility.',
330 'download_corpora', help='Download all corpora for a project.')
331 download_corpora_parser.add_argument('--fuzz-target',
334 'project', help='name of the project or path (external)')
338 reproduce_parser.add_argument('--valgrind',
341 reproduce_parser.add_argument('project',
342 help='name of the project or path (external)')
344 reproduce_parser.add_argument('testcase_path', help='path of local testcase')
353 shell_parser.add_argument('project',
354 help='name of the project or path (external)')
356 help='path of local source',
370 return os.path.exists(os.path.join('infra', 'base-images', image_name))
373 def check_project_exists(project): argument
374 """Checks if a project exists."""
375 if os.path.exists(project.path):
378 if project.is_external:
379 descriptive_project_name = project.path
381 descriptive_project_name = project.name
387 def _check_fuzzer_exists(project, fuzzer_name): argument
389 command = ['docker', 'run', '--rm']
390 command.extend(['-v', '%s:/out' % project.out])
393 command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])
405 def _get_absolute_path(path): argument
406 """Returns absolute path with user expansion."""
407 return os.path.abspath(os.path.expanduser(path))
415 def _get_project_build_subdir(project, subdir_name): argument
416 """Creates the |subdir_name| subdirectory of the |project| subdirectory in
417 |BUILD_DIR| and returns its path."""
418 directory = os.path.join(BUILD_DIR, subdir_name, project)
419 if not os.path.exists(directory):
425 def _get_out_dir(project=''): argument
426 """Creates and returns path to /out directory for the given project (if
428 return _get_project_build_subdir(project, 'out')
435 parser.add_argument('--architecture',
444 parser.add_argument('--engine',
454 '--sanitizer',
462 parser.add_argument('-e',
467 def build_image_impl(project, cache=True, pull=False): argument
469 image_name = project.name
472 image_project = 'oss-fuzz-base'
473 docker_build_dir = os.path.join(OSS_FUZZ_DIR, 'infra', 'base-images',
475 dockerfile_path = os.path.join(docker_build_dir, 'Dockerfile')
477 if not check_project_exists(project):
479 dockerfile_path = project.dockerfile_path
480 docker_build_dir = project.path
481 image_project = 'oss-fuzz'
483 if pull and not pull_images(project.language):
488 build_args.append('--no-cache')
491 '-t',
492 'gcr.io/%s/%s' % (image_project, image_name), '--file', dockerfile_path
500 return sum([['-e', v] for v in env_list], [])
503 def workdir_from_lines(lines, default='/src'):
509 workdir = workdir.replace('$SRC', '/src')
511 if not os.path.isabs(workdir):
512 workdir = os.path.join('/src', workdir)
514 return os.path.normpath(workdir)
519 def _workdir_from_dockerfile(project): argument
520 """Parses WORKDIR from the Dockerfile for the given project."""
521 with open(project.dockerfile_path) as file_handle:
524 return workdir_from_lines(lines, default=os.path.join('/src', project.name))
529 command = ['docker', 'run', '--rm', '--privileged']
533 command.append('-i')
582 logging.error('Incompatible arguments --pull and --no-pull.')
599 if build_image_impl(args.project, cache=args.cache, pull=pull):
605 def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
606 project, argument
615 if not build_image_impl(project):
621 # Clean old and possibly conflicting artifacts in project's out directory.
623 '-v',
624 '%s:/out' % project.out, '-t',
625 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /out/*'
629 '-v',
630 '%s:/work' % project.work, '-t',
631 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /work/*'
635 logging.info('Keeping existing build artifacts as-is (if any).')
644 if project.language:
645 env.append('FUZZING_LANGUAGE=' + project.language)
650 command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env)
652 workdir = _workdir_from_dockerfile(project)
655 '-v',
659 if workdir == '/src':
660 logging.error('Cannot use local checkout with "WORKDIR: /src".')
664 '-v',
669 '-v',
670 '%s:/out' % project.out, '-v',
671 '%s:/work' % project.work, '-t',
672 'gcr.io/oss-fuzz/%s' % project.name
685 return build_fuzzers_impl(args.project,
704 if not check_project_exists(args.project):
708 not _check_fuzzer_exists(args.project, args.fuzzer_name)):
711 fuzzing_language = args.project.language
714 logging.warning('Language not specified in project.yaml. Defaulting to %s.',
728 '-v', '%s:/out' % args.project.out, '-t', BASE_RUNNER_IMAGE
745 def _get_fuzz_targets(project): argument
746 """Returns names of fuzz targest build in the project's /out directory."""
748 for name in os.listdir(project.out):
749 if name.startswith('afl-'):
753 if name == 'llvm-symbolizer':
756 path = os.path.join(project.out, name)
759 if os.path.isfile(path) and (os.stat(path).st_mode & 0o111):
765 def _get_latest_corpus(project, fuzz_target, base_corpus_dir): argument
767 corpus_dir = os.path.join(base_corpus_dir, fuzz_target)
768 if not os.path.exists(corpus_dir):
771 if not fuzz_target.startswith(project.name + '_'):
772 fuzz_target = '%s_%s' % (project.name, fuzz_target)
774 corpus_backup_url = CORPUS_BACKUP_URL_FORMAT.format(project_name=project.name,
779 # asking for two-factor authentication.
789 latest_backup_url = output.splitlines()[-1]
791 command = ['gsutil', '-q', 'cp', latest_backup_url, archive_path]
794 command = ['unzip', '-q', '-o', archive_path, '-d', corpus_dir]
799 corpus_url = CORPUS_URL_FORMAT.format(project_name=project.name,
801 command = ['gsutil', '-m', '-q', 'rsync', '-R', corpus_url, corpus_dir]
806 """Downloads most recent corpora from GCS for the given project."""
807 if not check_project_exists(args.project):
812 subprocess.check_call(['gsutil', '--version'], stdout=stdout)
821 fuzz_targets = _get_fuzz_targets(args.project)
823 corpus_dir = args.project.corpus
827 _get_latest_corpus(args.project, fuzz_target, corpus_dir)
829 except Exception as error: # pylint:disable=broad-except
834 logging.info('Downloading corpora for %s project to %s.', args.project.name,
844 '--corpus-dir requires specifying a particular fuzz target using '
845 '--fuzz-target')
848 if not check_project_exists(args.project):
851 if args.project.language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT:
853 'Project is written in %s, coverage for it is not supported yet.',
854 args.project.language)
858 not args.project.is_external):
864 'FUZZING_LANGUAGE=%s' % args.project.language,
865 'PROJECT=%s' % args.project.name,
875 '-p',
880 if not os.path.exists(args.corpus_dir):
881 logging.error('The path provided in --corpus-dir argument does not '
884 corpus_dir = os.path.realpath(args.corpus_dir)
885 run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)])
887 run_args.extend(['-v', '%s:/corpus' % args.project.corpus])
890 '-v',
891 '%s:/out' % args.project.out,
892 '-t',
911 if not check_project_exists(args.project):
914 if not _check_fuzzer_exists(args.project, args.fuzzer_name):
929 if not os.path.exists(args.corpus_dir):
930 logging.error('The path provided in --corpus-dir argument does not exist')
932 corpus_dir = os.path.realpath(args.corpus_dir)
934 '-v',
940 '-v',
941 '%s:/out' % args.project.out,
942 '-t',
952 """Reproduces a specific test case from a specific project."""
953 return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e,
957 def reproduce_impl( # pylint: disable=too-many-arguments
958 project, argument
967 if not check_project_exists(project):
970 if not _check_fuzzer_exists(project, fuzzer_name):
975 image_name = 'base-runner'
978 debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full'
981 image_name = 'base-runner-debug'
988 '-v',
989 '%s:/out' % project.out,
990 '-v',
992 '-t',
993 'gcr.io/oss-fuzz-base/%s' % image_name,
996 '-runs=100',
1003 """Validates |project_name| is a valid OSS-Fuzz project name."""
1006 'Project name needs to be less than or equal to %d characters.',
1011 logging.info('Invalid project name: %s.', project_name)
1019 logging.error('Invalid project language %s.', language)
1027 Suitable for OSS-Fuzz and external projects."""
1042 file_path = os.path.join(directory, filename)
1051 """Generates empty project files."""
1052 return _generate_impl(args.project, args.language)
1063 return 'base-builder'
1064 return 'base-builder-{language}'.format(language=language)
1067 def _generate_impl(project, language): argument
1069 if project.is_external:
1070 # External project.
1073 # Internal project.
1074 if not _validate_project_name(project.name):
1081 directory = project.build_integration_path
1088 'project_name': project.name,
1100 if not build_image_impl(args.project):
1109 if args.project.name != 'base-runner-debug':
1110 env.append('FUZZING_LANGUAGE=' + args.project.language)
1115 if is_base_image(args.project.name):
1116 image_project = 'oss-fuzz-base'
1119 image_project = 'oss-fuzz'
1120 out_dir = args.project.out
1125 '-v',
1126 '%s:%s' % (_get_absolute_path(args.source_path), '/src'),
1130 '-v',
1131 '%s:/out' % out_dir, '-v',
1132 '%s:/work' % args.project.work, '-t',
1133 'gcr.io/%s/%s' % (image_project, args.project.name), '/bin/bash'