• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17"""Helper script for OSS-Fuzz users. Can do common tasks like building
18projects/fuzzers, running them etc."""
19
20from __future__ import print_function
21from multiprocessing.dummy import Pool as ThreadPool
22import argparse
23import datetime
24import errno
25import logging
26import os
27import pipes
28import re
29import subprocess
30import sys
31import templates
32
33import constants
34
35OSS_FUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
36BUILD_DIR = os.path.join(OSS_FUZZ_DIR, 'build')
37
38BASE_RUNNER_IMAGE = 'gcr.io/oss-fuzz-base/base-runner'
39
40BASE_IMAGES = {
41    'generic': [
42        'gcr.io/oss-fuzz-base/base-image',
43        'gcr.io/oss-fuzz-base/base-clang',
44        'gcr.io/oss-fuzz-base/base-builder',
45        BASE_RUNNER_IMAGE,
46        'gcr.io/oss-fuzz-base/base-runner-debug',
47    ],
48    'go': ['gcr.io/oss-fuzz-base/base-builder-go'],
49    'jvm': ['gcr.io/oss-fuzz-base/base-builder-jvm'],
50    'python': ['gcr.io/oss-fuzz-base/base-builder-python'],
51    'rust': ['gcr.io/oss-fuzz-base/base-builder-rust'],
52    'swift': ['gcr.io/oss-fuzz-base/base-builder-swift'],
53}
54
55VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
56MAX_PROJECT_NAME_LENGTH = 26
57
58CORPUS_URL_FORMAT = (
59    'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
60    '{fuzz_target}/')
61CORPUS_BACKUP_URL_FORMAT = (
62    'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
63    'libFuzzer/{fuzz_target}/')
64
65LANGUAGE_REGEX = re.compile(r'[^\s]+')
66PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)')
67
68WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
69
70LANGUAGES_WITH_BUILDER_IMAGES = {'go', 'jvm', 'python', 'rust', 'swift'}
71
72if sys.version_info[0] >= 3:
73  raw_input = input  # pylint: disable=invalid-name
74
75# pylint: disable=too-many-lines
76
77
78class Project:
79  """Class representing a project that is in OSS-Fuzz or an external project
80  (ClusterFuzzLite user)."""
81
82  def __init__(
83      self,
84      project_name_or_path,
85      is_external=False,
86      build_integration_path=constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH):
87    self.is_external = is_external
88    if self.is_external:
89      self.path = os.path.abspath(project_name_or_path)
90      self.name = os.path.basename(self.path)
91      self.build_integration_path = os.path.join(self.path,
92                                                 build_integration_path)
93    else:
94      self.name = project_name_or_path
95      self.path = os.path.join(OSS_FUZZ_DIR, 'projects', self.name)
96      self.build_integration_path = self.path
97
98  @property
99  def dockerfile_path(self):
100    """Returns path to the project Dockerfile."""
101    return os.path.join(self.build_integration_path, 'Dockerfile')
102
103  @property
104  def language(self):
105    """Returns project language."""
106    if self.is_external:
107      # TODO(metzman): Handle this properly.
108      return constants.DEFAULT_LANGUAGE
109
110    project_yaml_path = os.path.join(self.path, 'project.yaml')
111    with open(project_yaml_path) as file_handle:
112      content = file_handle.read()
113      for line in content.splitlines():
114        match = PROJECT_LANGUAGE_REGEX.match(line)
115        if match:
116          return match.group(1)
117
118    logging.warning('Language not specified in project.yaml.')
119    return None
120
121  @property
122  def out(self):
123    """Returns the out dir for the project. Creates it if needed."""
124    return _get_out_dir(self.name)
125
126  @property
127  def work(self):
128    """Returns the out dir for the project. Creates it if needed."""
129    return _get_project_build_subdir(self.name, 'work')
130
131  @property
132  def corpus(self):
133    """Returns the out dir for the project. Creates it if needed."""
134    return _get_project_build_subdir(self.name, 'corpus')
135
136
137def main():  # pylint: disable=too-many-branches,too-many-return-statements
138  """Gets subcommand from program arguments and does it. Returns 0 on success 1
139  on error."""
140  logging.basicConfig(level=logging.INFO)
141
142  parser = get_parser()
143  args = parse_args(parser)
144
145  # Note: this has to happen after parse_args above as parse_args needs to know
146  # the original CWD for external projects.
147  os.chdir(OSS_FUZZ_DIR)
148  if not os.path.exists(BUILD_DIR):
149    os.mkdir(BUILD_DIR)
150
151  # We have different default values for `sanitizer` depending on the `engine`.
152  # Some commands do not have `sanitizer` argument, so `hasattr` is necessary.
153  if hasattr(args, 'sanitizer') and not args.sanitizer:
154    if args.engine == 'dataflow':
155      args.sanitizer = 'dataflow'
156    else:
157      args.sanitizer = constants.DEFAULT_SANITIZER
158
159  if args.command == 'generate':
160    result = generate(args)
161  elif args.command == 'build_image':
162    result = build_image(args)
163  elif args.command == 'build_fuzzers':
164    result = build_fuzzers(args)
165  elif args.command == 'check_build':
166    result = check_build(args)
167  elif args.command == 'download_corpora':
168    result = download_corpora(args)
169  elif args.command == 'run_fuzzer':
170    result = run_fuzzer(args)
171  elif args.command == 'coverage':
172    result = coverage(args)
173  elif args.command == 'reproduce':
174    result = reproduce(args)
175  elif args.command == 'shell':
176    result = shell(args)
177  elif args.command == 'pull_images':
178    result = pull_images()
179  else:
180    # Print help string if no arguments provided.
181    parser.print_help()
182    result = False
183  return bool_to_retcode(result)
184
185
186def bool_to_retcode(boolean):
187  """Returns 0 if |boolean| is Truthy, 0 is the standard return code for a
188  successful process execution. Returns 1 otherwise, indicating the process
189  failed."""
190  return 0 if boolean else 1
191
192
193def parse_args(parser, args=None):
194  """Parses |args| using |parser| and returns parsed args. Also changes
195  |args.build_integration_path| to have correct default behavior."""
196  # Use default argument None for args so that in production, argparse does its
197  # normal behavior, but unittesting is easier.
198  parsed_args = parser.parse_args(args)
199  project = getattr(parsed_args, 'project', None)
200  if not project:
201    return parsed_args
202
203  # Use hacky method for extracting attributes so that ShellTest works.
204  # TODO(metzman): Fix this.
205  is_external = getattr(parsed_args, 'external', False)
206  parsed_args.project = Project(parsed_args.project, is_external)
207  return parsed_args
208
209
210def _add_external_project_args(parser):
211  parser.add_argument(
212      '--external',
213      help='Is project external?',
214      default=False,
215      action='store_true',
216  )
217
218
219def get_parser():  # pylint: disable=too-many-statements
220  """Returns an argparse parser."""
221  parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
222  subparsers = parser.add_subparsers(dest='command')
223
224  generate_parser = subparsers.add_parser(
225      'generate', help='Generate files for new project.')
226  generate_parser.add_argument('project')
227  generate_parser.add_argument(
228      '--language',
229      default=constants.DEFAULT_LANGUAGE,
230      choices=['c', 'c++', 'rust', 'go', 'jvm', 'swift', 'python'],
231      help='Project language.')
232  _add_external_project_args(generate_parser)
233
234  build_image_parser = subparsers.add_parser('build_image',
235                                             help='Build an image.')
236  build_image_parser.add_argument('project')
237  build_image_parser.add_argument('--pull',
238                                  action='store_true',
239                                  help='Pull latest base image.')
240  build_image_parser.add_argument('--cache',
241                                  action='store_true',
242                                  default=False,
243                                  help='Use docker cache when building image.')
244  build_image_parser.add_argument('--no-pull',
245                                  action='store_true',
246                                  help='Do not pull latest base image.')
247  _add_external_project_args(build_image_parser)
248
249  build_fuzzers_parser = subparsers.add_parser(
250      'build_fuzzers', help='Build fuzzers for a project.')
251  _add_architecture_args(build_fuzzers_parser)
252  _add_engine_args(build_fuzzers_parser)
253  _add_sanitizer_args(build_fuzzers_parser)
254  _add_environment_args(build_fuzzers_parser)
255  _add_external_project_args(build_fuzzers_parser)
256  build_fuzzers_parser.add_argument('project')
257  build_fuzzers_parser.add_argument('source_path',
258                                    help='path of local source',
259                                    nargs='?')
260  build_fuzzers_parser.add_argument('--mount_path',
261                                    dest='mount_path',
262                                    help='path to mount local source in '
263                                    '(defaults to WORKDIR)')
264  build_fuzzers_parser.add_argument('--clean',
265                                    dest='clean',
266                                    action='store_true',
267                                    help='clean existing artifacts.')
268  build_fuzzers_parser.add_argument('--no-clean',
269                                    dest='clean',
270                                    action='store_false',
271                                    help='do not clean existing artifacts '
272                                    '(default).')
273  build_fuzzers_parser.set_defaults(clean=False)
274
275  check_build_parser = subparsers.add_parser(
276      'check_build', help='Checks that fuzzers execute without errors.')
277  _add_architecture_args(check_build_parser)
278  _add_engine_args(check_build_parser, choices=constants.ENGINES)
279  _add_sanitizer_args(check_build_parser, choices=constants.SANITIZERS)
280  _add_environment_args(check_build_parser)
281  check_build_parser.add_argument('project',
282                                  help='name of the project or path (external)')
283  check_build_parser.add_argument('fuzzer_name',
284                                  help='name of the fuzzer',
285                                  nargs='?')
286  _add_external_project_args(check_build_parser)
287
288  run_fuzzer_parser = subparsers.add_parser(
289      'run_fuzzer', help='Run a fuzzer in the emulated fuzzing environment.')
290  _add_engine_args(run_fuzzer_parser)
291  _add_sanitizer_args(run_fuzzer_parser)
292  _add_environment_args(run_fuzzer_parser)
293  _add_external_project_args(run_fuzzer_parser)
294  run_fuzzer_parser.add_argument(
295      '--corpus-dir', help='directory to store corpus for the fuzz target')
296  run_fuzzer_parser.add_argument('project',
297                                 help='name of the project or path (external)')
298  run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
299  run_fuzzer_parser.add_argument('fuzzer_args',
300                                 help='arguments to pass to the fuzzer',
301                                 nargs=argparse.REMAINDER)
302
303  coverage_parser = subparsers.add_parser(
304      'coverage', help='Generate code coverage report for the project.')
305  coverage_parser.add_argument('--no-corpus-download',
306                               action='store_true',
307                               help='do not download corpus backup from '
308                               'OSS-Fuzz; use corpus located in '
309                               'build/corpus/<project>/<fuzz_target>/')
310  coverage_parser.add_argument('--port',
311                               default='8008',
312                               help='specify port for'
313                               ' a local HTTP server rendering coverage report')
314  coverage_parser.add_argument('--fuzz-target',
315                               help='specify name of a fuzz '
316                               'target to be run for generating coverage '
317                               'report')
318  coverage_parser.add_argument('--corpus-dir',
319                               help='specify location of corpus'
320                               ' to be used (requires --fuzz-target argument)')
321  coverage_parser.add_argument('project',
322                               help='name of the project or path (external)')
323  coverage_parser.add_argument('extra_args',
324                               help='additional arguments to '
325                               'pass to llvm-cov utility.',
326                               nargs='*')
327  _add_external_project_args(coverage_parser)
328
329  download_corpora_parser = subparsers.add_parser(
330      'download_corpora', help='Download all corpora for a project.')
331  download_corpora_parser.add_argument('--fuzz-target',
332                                       help='specify name of a fuzz target')
333  download_corpora_parser.add_argument(
334      'project', help='name of the project or path (external)')
335
336  reproduce_parser = subparsers.add_parser('reproduce',
337                                           help='Reproduce a crash.')
338  reproduce_parser.add_argument('--valgrind',
339                                action='store_true',
340                                help='run with valgrind')
341  reproduce_parser.add_argument('project',
342                                help='name of the project or path (external)')
343  reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
344  reproduce_parser.add_argument('testcase_path', help='path of local testcase')
345  reproduce_parser.add_argument('fuzzer_args',
346                                help='arguments to pass to the fuzzer',
347                                nargs=argparse.REMAINDER)
348  _add_environment_args(reproduce_parser)
349  _add_external_project_args(reproduce_parser)
350
351  shell_parser = subparsers.add_parser(
352      'shell', help='Run /bin/bash within the builder container.')
353  shell_parser.add_argument('project',
354                            help='name of the project or path (external)')
355  shell_parser.add_argument('source_path',
356                            help='path of local source',
357                            nargs='?')
358  _add_architecture_args(shell_parser)
359  _add_engine_args(shell_parser)
360  _add_sanitizer_args(shell_parser)
361  _add_environment_args(shell_parser)
362  _add_external_project_args(shell_parser)
363
364  subparsers.add_parser('pull_images', help='Pull base images.')
365  return parser
366
367
368def is_base_image(image_name):
369  """Checks if the image name is a base image."""
370  return os.path.exists(os.path.join('infra', 'base-images', image_name))
371
372
373def check_project_exists(project):
374  """Checks if a project exists."""
375  if os.path.exists(project.path):
376    return True
377
378  if project.is_external:
379    descriptive_project_name = project.path
380  else:
381    descriptive_project_name = project.name
382
383  logging.error('"%s" does not exist.', descriptive_project_name)
384  return False
385
386
387def _check_fuzzer_exists(project, fuzzer_name):
388  """Checks if a fuzzer exists."""
389  command = ['docker', 'run', '--rm']
390  command.extend(['-v', '%s:/out' % project.out])
391  command.append(BASE_RUNNER_IMAGE)
392
393  command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])
394
395  try:
396    subprocess.check_call(command)
397  except subprocess.CalledProcessError:
398    logging.error('%s does not seem to exist. Please run build_fuzzers first.',
399                  fuzzer_name)
400    return False
401
402  return True
403
404
405def _get_absolute_path(path):
406  """Returns absolute path with user expansion."""
407  return os.path.abspath(os.path.expanduser(path))
408
409
410def _get_command_string(command):
411  """Returns a shell escaped command string."""
412  return ' '.join(pipes.quote(part) for part in command)
413
414
415def _get_project_build_subdir(project, subdir_name):
416  """Creates the |subdir_name| subdirectory of the |project| subdirectory in
417  |BUILD_DIR| and returns its path."""
418  directory = os.path.join(BUILD_DIR, subdir_name, project)
419  if not os.path.exists(directory):
420    os.makedirs(directory)
421
422  return directory
423
424
425def _get_out_dir(project=''):
426  """Creates and returns path to /out directory for the given project (if
427  specified)."""
428  return _get_project_build_subdir(project, 'out')
429
430
431def _add_architecture_args(parser, choices=None):
432  """Adds common architecture args."""
433  if choices is None:
434    choices = constants.ARCHITECTURES
435  parser.add_argument('--architecture',
436                      default=constants.DEFAULT_ARCHITECTURE,
437                      choices=choices)
438
439
440def _add_engine_args(parser, choices=None):
441  """Adds common engine args."""
442  if choices is None:
443    choices = constants.ENGINES
444  parser.add_argument('--engine',
445                      default=constants.DEFAULT_ENGINE,
446                      choices=choices)
447
448
449def _add_sanitizer_args(parser, choices=None):
450  """Adds common sanitizer args."""
451  if choices is None:
452    choices = constants.SANITIZERS
453  parser.add_argument(
454      '--sanitizer',
455      default=None,
456      choices=choices,
457      help='the default is "address"; "dataflow" for "dataflow" engine')
458
459
460def _add_environment_args(parser):
461  """Adds common environment args."""
462  parser.add_argument('-e',
463                      action='append',
464                      help="set environment variable e.g. VAR=value")
465
466
467def build_image_impl(project, cache=True, pull=False):
468  """Builds image."""
469  image_name = project.name
470
471  if is_base_image(image_name):
472    image_project = 'oss-fuzz-base'
473    docker_build_dir = os.path.join(OSS_FUZZ_DIR, 'infra', 'base-images',
474                                    image_name)
475    dockerfile_path = os.path.join(docker_build_dir, 'Dockerfile')
476  else:
477    if not check_project_exists(project):
478      return False
479    dockerfile_path = project.dockerfile_path
480    docker_build_dir = project.path
481    image_project = 'oss-fuzz'
482
483  if pull and not pull_images(project.language):
484    return False
485
486  build_args = []
487  if not cache:
488    build_args.append('--no-cache')
489
490  build_args += [
491      '-t',
492      'gcr.io/%s/%s' % (image_project, image_name), '--file', dockerfile_path
493  ]
494  build_args.append(docker_build_dir)
495  return docker_build(build_args)
496
497
498def _env_to_docker_args(env_list):
499  """Turns envirnoment variable list into docker arguments."""
500  return sum([['-e', v] for v in env_list], [])
501
502
503def workdir_from_lines(lines, default='/src'):
504  """Gets the WORKDIR from the given lines."""
505  for line in reversed(lines):  # reversed to get last WORKDIR.
506    match = re.match(WORKDIR_REGEX, line)
507    if match:
508      workdir = match.group(1)
509      workdir = workdir.replace('$SRC', '/src')
510
511      if not os.path.isabs(workdir):
512        workdir = os.path.join('/src', workdir)
513
514      return os.path.normpath(workdir)
515
516  return default
517
518
519def _workdir_from_dockerfile(project):
520  """Parses WORKDIR from the Dockerfile for the given project."""
521  with open(project.dockerfile_path) as file_handle:
522    lines = file_handle.readlines()
523
524  return workdir_from_lines(lines, default=os.path.join('/src', project.name))
525
526
527def docker_run(run_args, print_output=True):
528  """Calls `docker run`."""
529  command = ['docker', 'run', '--rm', '--privileged']
530
531  # Support environments with a TTY.
532  if sys.stdin.isatty():
533    command.append('-i')
534
535  command.extend(run_args)
536
537  logging.info('Running: %s.', _get_command_string(command))
538  stdout = None
539  if not print_output:
540    stdout = open(os.devnull, 'w')
541
542  try:
543    subprocess.check_call(command, stdout=stdout, stderr=subprocess.STDOUT)
544  except subprocess.CalledProcessError:
545    return False
546
547  return True
548
549
550def docker_build(build_args):
551  """Calls `docker build`."""
552  command = ['docker', 'build']
553  command.extend(build_args)
554  logging.info('Running: %s.', _get_command_string(command))
555
556  try:
557    subprocess.check_call(command)
558  except subprocess.CalledProcessError:
559    logging.error('Docker build failed.')
560    return False
561
562  return True
563
564
565def docker_pull(image):
566  """Call `docker pull`."""
567  command = ['docker', 'pull', image]
568  logging.info('Running: %s', _get_command_string(command))
569
570  try:
571    subprocess.check_call(command)
572  except subprocess.CalledProcessError:
573    logging.error('Docker pull failed.')
574    return False
575
576  return True
577
578
579def build_image(args):
580  """Builds docker image."""
581  if args.pull and args.no_pull:
582    logging.error('Incompatible arguments --pull and --no-pull.')
583    return False
584
585  if args.pull:
586    pull = True
587  elif args.no_pull:
588    pull = False
589  else:
590    y_or_n = raw_input('Pull latest base images (compiler/runtime)? (y/N): ')
591    pull = y_or_n.lower() == 'y'
592
593  if pull:
594    logging.info('Pulling latest base images...')
595  else:
596    logging.info('Using cached base images...')
597
598  # If build_image is called explicitly, don't use cache.
599  if build_image_impl(args.project, cache=args.cache, pull=pull):
600    return True
601
602  return False
603
604
605def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
606    project,
607    clean,
608    engine,
609    sanitizer,
610    architecture,
611    env_to_add,
612    source_path,
613    mount_path=None):
614  """Builds fuzzers."""
615  if not build_image_impl(project):
616    return False
617
618  if clean:
619    logging.info('Cleaning existing build artifacts.')
620
621    # Clean old and possibly conflicting artifacts in project's out directory.
622    docker_run([
623        '-v',
624        '%s:/out' % project.out, '-t',
625        'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /out/*'
626    ])
627
628    docker_run([
629        '-v',
630        '%s:/work' % project.work, '-t',
631        'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /work/*'
632    ])
633
634  else:
635    logging.info('Keeping existing build artifacts as-is (if any).')
636  env = [
637      'FUZZING_ENGINE=' + engine,
638      'SANITIZER=' + sanitizer,
639      'ARCHITECTURE=' + architecture,
640  ]
641
642  _add_oss_fuzz_ci_if_needed(env)
643
644  if project.language:
645    env.append('FUZZING_LANGUAGE=' + project.language)
646
647  if env_to_add:
648    env += env_to_add
649
650  command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env)
651  if source_path:
652    workdir = _workdir_from_dockerfile(project)
653    if mount_path:
654      command += [
655          '-v',
656          '%s:%s' % (_get_absolute_path(source_path), mount_path),
657      ]
658    else:
659      if workdir == '/src':
660        logging.error('Cannot use local checkout with "WORKDIR: /src".')
661        return False
662
663      command += [
664          '-v',
665          '%s:%s' % (_get_absolute_path(source_path), workdir),
666      ]
667
668  command += [
669      '-v',
670      '%s:/out' % project.out, '-v',
671      '%s:/work' % project.work, '-t',
672      'gcr.io/oss-fuzz/%s' % project.name
673  ]
674
675  result = docker_run(command)
676  if not result:
677    logging.error('Building fuzzers failed.')
678    return False
679
680  return True
681
682
683def build_fuzzers(args):
684  """Builds fuzzers."""
685  return build_fuzzers_impl(args.project,
686                            args.clean,
687                            args.engine,
688                            args.sanitizer,
689                            args.architecture,
690                            args.e,
691                            args.source_path,
692                            mount_path=args.mount_path)
693
694
695def _add_oss_fuzz_ci_if_needed(env):
696  """Adds value of |OSS_FUZZ_CI| environment variable to |env| if it is set."""
697  oss_fuzz_ci = os.getenv('OSS_FUZZ_CI')
698  if oss_fuzz_ci:
699    env.append('OSS_FUZZ_CI=' + oss_fuzz_ci)
700
701
702def check_build(args):
703  """Checks that fuzzers in the container execute without errors."""
704  if not check_project_exists(args.project):
705    return False
706
707  if (args.fuzzer_name and
708      not _check_fuzzer_exists(args.project, args.fuzzer_name)):
709    return False
710
711  fuzzing_language = args.project.language
712  if not fuzzing_language:
713    fuzzing_language = constants.DEFAULT_LANGUAGE
714    logging.warning('Language not specified in project.yaml. Defaulting to %s.',
715                    fuzzing_language)
716
717  env = [
718      'FUZZING_ENGINE=' + args.engine,
719      'SANITIZER=' + args.sanitizer,
720      'ARCHITECTURE=' + args.architecture,
721      'FUZZING_LANGUAGE=' + fuzzing_language,
722  ]
723  _add_oss_fuzz_ci_if_needed(env)
724  if args.e:
725    env += args.e
726
727  run_args = _env_to_docker_args(env) + [
728      '-v', '%s:/out' % args.project.out, '-t', BASE_RUNNER_IMAGE
729  ]
730
731  if args.fuzzer_name:
732    run_args += ['test_one.py', args.fuzzer_name]
733  else:
734    run_args.append('test_all.py')
735
736  result = docker_run(run_args)
737  if result:
738    logging.info('Check build passed.')
739  else:
740    logging.error('Check build failed.')
741
742  return result
743
744
745def _get_fuzz_targets(project):
746  """Returns names of fuzz targest build in the project's /out directory."""
747  fuzz_targets = []
748  for name in os.listdir(project.out):
749    if name.startswith('afl-'):
750      continue
751    if name.startswith('jazzer_'):
752      continue
753    if name == 'llvm-symbolizer':
754      continue
755
756    path = os.path.join(project.out, name)
757    # Python and JVM fuzz targets are only executable for the root user, so
758    # we can't use os.access.
759    if os.path.isfile(path) and (os.stat(path).st_mode & 0o111):
760      fuzz_targets.append(name)
761
762  return fuzz_targets
763
764
765def _get_latest_corpus(project, fuzz_target, base_corpus_dir):
766  """Downloads the latest corpus for the given fuzz target."""
767  corpus_dir = os.path.join(base_corpus_dir, fuzz_target)
768  if not os.path.exists(corpus_dir):
769    os.makedirs(corpus_dir)
770
771  if not fuzz_target.startswith(project.name + '_'):
772    fuzz_target = '%s_%s' % (project.name, fuzz_target)
773
774  corpus_backup_url = CORPUS_BACKUP_URL_FORMAT.format(project_name=project.name,
775                                                      fuzz_target=fuzz_target)
776  command = ['gsutil', 'ls', corpus_backup_url]
777
778  # Don't capture stderr. We want it to print in real time, in case gsutil is
779  # asking for two-factor authentication.
780  corpus_listing = subprocess.Popen(command, stdout=subprocess.PIPE)
781  output, _ = corpus_listing.communicate()
782
783  # Some fuzz targets (e.g. new ones) may not have corpus yet, just skip those.
784  if corpus_listing.returncode:
785    logging.warning('Corpus for %s not found:\n', fuzz_target)
786    return
787
788  if output:
789    latest_backup_url = output.splitlines()[-1]
790    archive_path = corpus_dir + '.zip'
791    command = ['gsutil', '-q', 'cp', latest_backup_url, archive_path]
792    subprocess.check_call(command)
793
794    command = ['unzip', '-q', '-o', archive_path, '-d', corpus_dir]
795    subprocess.check_call(command)
796    os.remove(archive_path)
797  else:
798    # Sync the working corpus copy if a minimized backup is not available.
799    corpus_url = CORPUS_URL_FORMAT.format(project_name=project.name,
800                                          fuzz_target=fuzz_target)
801    command = ['gsutil', '-m', '-q', 'rsync', '-R', corpus_url, corpus_dir]
802    subprocess.check_call(command)
803
804
805def download_corpora(args):
806  """Downloads most recent corpora from GCS for the given project."""
807  if not check_project_exists(args.project):
808    return False
809
810  try:
811    with open(os.devnull, 'w') as stdout:
812      subprocess.check_call(['gsutil', '--version'], stdout=stdout)
813  except OSError:
814    logging.error('gsutil not found. Please install it from '
815                  'https://cloud.google.com/storage/docs/gsutil_install')
816    return False
817
818  if args.fuzz_target:
819    fuzz_targets = [args.fuzz_target]
820  else:
821    fuzz_targets = _get_fuzz_targets(args.project)
822
823  corpus_dir = args.project.corpus
824
825  def _download_for_single_target(fuzz_target):
826    try:
827      _get_latest_corpus(args.project, fuzz_target, corpus_dir)
828      return True
829    except Exception as error:  # pylint:disable=broad-except
830      logging.error('Corpus download for %s failed: %s.', fuzz_target,
831                    str(error))
832      return False
833
834  logging.info('Downloading corpora for %s project to %s.', args.project.name,
835               corpus_dir)
836  thread_pool = ThreadPool()
837  return all(thread_pool.map(_download_for_single_target, fuzz_targets))
838
839
840def coverage(args):
841  """Generates code coverage using clang source based code coverage."""
842  if args.corpus_dir and not args.fuzz_target:
843    logging.error(
844        '--corpus-dir requires specifying a particular fuzz target using '
845        '--fuzz-target')
846    return False
847
848  if not check_project_exists(args.project):
849    return False
850
851  if args.project.language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT:
852    logging.error(
853        'Project is written in %s, coverage for it is not supported yet.',
854        args.project.language)
855    return False
856
857  if (not args.no_corpus_download and not args.corpus_dir and
858      not args.project.is_external):
859    if not download_corpora(args):
860      return False
861
862  env = [
863      'FUZZING_ENGINE=libfuzzer',
864      'FUZZING_LANGUAGE=%s' % args.project.language,
865      'PROJECT=%s' % args.project.name,
866      'SANITIZER=coverage',
867      'HTTP_PORT=%s' % args.port,
868      'COVERAGE_EXTRA_ARGS=%s' % ' '.join(args.extra_args),
869  ]
870
871  run_args = _env_to_docker_args(env)
872
873  if args.port:
874    run_args.extend([
875        '-p',
876        '%s:%s' % (args.port, args.port),
877    ])
878
879  if args.corpus_dir:
880    if not os.path.exists(args.corpus_dir):
881      logging.error('The path provided in --corpus-dir argument does not '
882                    'exist.')
883      return False
884    corpus_dir = os.path.realpath(args.corpus_dir)
885    run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)])
886  else:
887    run_args.extend(['-v', '%s:/corpus' % args.project.corpus])
888
889  run_args.extend([
890      '-v',
891      '%s:/out' % args.project.out,
892      '-t',
893      BASE_RUNNER_IMAGE,
894  ])
895
896  run_args.append('coverage')
897  if args.fuzz_target:
898    run_args.append(args.fuzz_target)
899
900  result = docker_run(run_args)
901  if result:
902    logging.info('Successfully generated clang code coverage report.')
903  else:
904    logging.error('Failed to generate clang code coverage report.')
905
906  return result
907
908
909def run_fuzzer(args):
910  """Runs a fuzzer in the container."""
911  if not check_project_exists(args.project):
912    return False
913
914  if not _check_fuzzer_exists(args.project, args.fuzzer_name):
915    return False
916
917  env = [
918      'FUZZING_ENGINE=' + args.engine,
919      'SANITIZER=' + args.sanitizer,
920      'RUN_FUZZER_MODE=interactive',
921  ]
922
923  if args.e:
924    env += args.e
925
926  run_args = _env_to_docker_args(env)
927
928  if args.corpus_dir:
929    if not os.path.exists(args.corpus_dir):
930      logging.error('The path provided in --corpus-dir argument does not exist')
931      return False
932    corpus_dir = os.path.realpath(args.corpus_dir)
933    run_args.extend([
934        '-v',
935        '{corpus_dir}:/tmp/{fuzzer}_corpus'.format(corpus_dir=corpus_dir,
936                                                   fuzzer=args.fuzzer_name)
937    ])
938
939  run_args.extend([
940      '-v',
941      '%s:/out' % args.project.out,
942      '-t',
943      BASE_RUNNER_IMAGE,
944      'run_fuzzer',
945      args.fuzzer_name,
946  ] + args.fuzzer_args)
947
948  return docker_run(run_args)
949
950
951def reproduce(args):
952  """Reproduces a specific test case from a specific project."""
953  return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e,
954                        args.fuzzer_args, args.testcase_path)
955
956
957def reproduce_impl(  # pylint: disable=too-many-arguments
958    project,
959    fuzzer_name,
960    valgrind,
961    env_to_add,
962    fuzzer_args,
963    testcase_path,
964    run_function=docker_run,
965    err_result=False):
966  """Reproduces a testcase in the container."""
967  if not check_project_exists(project):
968    return err_result
969
970  if not _check_fuzzer_exists(project, fuzzer_name):
971    return err_result
972
973  debugger = ''
974  env = []
975  image_name = 'base-runner'
976
977  if valgrind:
978    debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full'
979
980  if debugger:
981    image_name = 'base-runner-debug'
982    env += ['DEBUGGER=' + debugger]
983
984  if env_to_add:
985    env += env_to_add
986
987  run_args = _env_to_docker_args(env) + [
988      '-v',
989      '%s:/out' % project.out,
990      '-v',
991      '%s:/testcase' % _get_absolute_path(testcase_path),
992      '-t',
993      'gcr.io/oss-fuzz-base/%s' % image_name,
994      'reproduce',
995      fuzzer_name,
996      '-runs=100',
997  ] + fuzzer_args
998
999  return run_function(run_args)
1000
1001
1002def _validate_project_name(project_name):
1003  """Validates |project_name| is a valid OSS-Fuzz project name."""
1004  if len(project_name) > MAX_PROJECT_NAME_LENGTH:
1005    logging.error(
1006        'Project name needs to be less than or equal to %d characters.',
1007        MAX_PROJECT_NAME_LENGTH)
1008    return False
1009
1010  if not VALID_PROJECT_NAME_REGEX.match(project_name):
1011    logging.info('Invalid project name: %s.', project_name)
1012    return False
1013
1014  return True
1015
1016
1017def _validate_language(language):
1018  if not LANGUAGE_REGEX.match(language):
1019    logging.error('Invalid project language %s.', language)
1020    return False
1021
1022  return True
1023
1024
1025def _create_build_integration_directory(directory):
1026  """Returns True on successful creation of a build integration directory.
1027  Suitable for OSS-Fuzz and external projects."""
1028  try:
1029    os.makedirs(directory)
1030  except OSError as error:
1031    if error.errno != errno.EEXIST:
1032      raise
1033    logging.error('%s already exists.', directory)
1034    return False
1035  return True
1036
1037
1038def _template_project_file(filename, template, template_args, directory):
1039  """Templates |template| using |template_args| and writes the result to
1040  |directory|/|filename|. Sets the file to executable if |filename| is
1041  build.sh."""
1042  file_path = os.path.join(directory, filename)
1043  with open(file_path, 'w') as file_handle:
1044    file_handle.write(template % template_args)
1045
1046  if filename == 'build.sh':
1047    os.chmod(file_path, 0o755)
1048
1049
1050def generate(args):
1051  """Generates empty project files."""
1052  return _generate_impl(args.project, args.language)
1053
1054
1055def _get_current_datetime():
1056  """Returns this year. Needed for mocking."""
1057  return datetime.datetime.now()
1058
1059
1060def _base_builder_from_language(language):
1061  """Returns the base builder for the specified language."""
1062  if language not in LANGUAGES_WITH_BUILDER_IMAGES:
1063    return 'base-builder'
1064  return 'base-builder-{language}'.format(language=language)
1065
1066
1067def _generate_impl(project, language):
1068  """Implementation of generate(). Useful for testing."""
1069  if project.is_external:
1070    # External project.
1071    project_templates = templates.EXTERNAL_TEMPLATES
1072  else:
1073    # Internal project.
1074    if not _validate_project_name(project.name):
1075      return False
1076    project_templates = templates.TEMPLATES
1077
1078  if not _validate_language(language):
1079    return False
1080
1081  directory = project.build_integration_path
1082  if not _create_build_integration_directory(directory):
1083    return False
1084
1085  logging.info('Writing new files to: %s.', directory)
1086
1087  template_args = {
1088      'project_name': project.name,
1089      'base_builder': _base_builder_from_language(language),
1090      'language': language,
1091      'year': _get_current_datetime().year
1092  }
1093  for filename, template in project_templates.items():
1094    _template_project_file(filename, template, template_args, directory)
1095  return True
1096
1097
1098def shell(args):
1099  """Runs a shell within a docker image."""
1100  if not build_image_impl(args.project):
1101    return False
1102
1103  env = [
1104      'FUZZING_ENGINE=' + args.engine,
1105      'SANITIZER=' + args.sanitizer,
1106      'ARCHITECTURE=' + args.architecture,
1107  ]
1108
1109  if args.project.name != 'base-runner-debug':
1110    env.append('FUZZING_LANGUAGE=' + args.project.language)
1111
1112  if args.e:
1113    env += args.e
1114
1115  if is_base_image(args.project.name):
1116    image_project = 'oss-fuzz-base'
1117    out_dir = _get_out_dir()
1118  else:
1119    image_project = 'oss-fuzz'
1120    out_dir = args.project.out
1121
1122  run_args = _env_to_docker_args(env)
1123  if args.source_path:
1124    run_args.extend([
1125        '-v',
1126        '%s:%s' % (_get_absolute_path(args.source_path), '/src'),
1127    ])
1128
1129  run_args.extend([
1130      '-v',
1131      '%s:/out' % out_dir, '-v',
1132      '%s:/work' % args.project.work, '-t',
1133      'gcr.io/%s/%s' % (image_project, args.project.name), '/bin/bash'
1134  ])
1135
1136  docker_run(run_args)
1137  return True
1138
1139
1140def pull_images(language=None):
1141  """Pulls base images used to build projects in language lang (or all if lang
1142  is None)."""
1143  for base_image_lang, base_images in BASE_IMAGES.items():
1144    if (language is None or base_image_lang == 'generic' or
1145        base_image_lang == language):
1146      for base_image in base_images:
1147        if not docker_pull(base_image):
1148          return False
1149
1150  return True
1151
1152
1153if __name__ == '__main__':
1154  sys.exit(main())
1155