• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Utility module for Google Cloud Build scripts."""
2import base64
3import collections
4import os
5import requests
6import sys
7import time
8import urllib
9import urlparse
10
11from oauth2client.service_account import ServiceAccountCredentials
12
13BUILD_TIMEOUT = 12 * 60 * 60
14
15# Needed for reading public target.list.* files.
16GCS_URL_BASENAME = 'https://storage.googleapis.com/'
17
18GCS_UPLOAD_URL_FORMAT = '/{0}/{1}/{2}'
19
20# Where corpus backups can be downloaded from.
21CORPUS_BACKUP_URL = ('/{project}-backup.clusterfuzz-external.appspot.com/'
22                     'corpus/libFuzzer/{fuzzer}/latest.zip')
23
24# Cloud Builder has a limit of 100 build steps and 100 arguments for each step.
25CORPUS_DOWNLOAD_BATCH_SIZE = 100
26
27TARGETS_LIST_BASENAME = 'targets.list'
28
29EngineInfo = collections.namedtuple(
30    'EngineInfo',
31    ['upload_bucket', 'supported_sanitizers', 'supported_architectures'])
32
33ENGINE_INFO = {
34    'libfuzzer':
35        EngineInfo(upload_bucket='clusterfuzz-builds',
36                   supported_sanitizers=['address', 'memory', 'undefined'],
37                   supported_architectures=['x86_64', 'i386']),
38    'afl':
39        EngineInfo(upload_bucket='clusterfuzz-builds-afl',
40                   supported_sanitizers=['address'],
41                   supported_architectures=['x86_64']),
42    'honggfuzz':
43        EngineInfo(upload_bucket='clusterfuzz-builds-honggfuzz',
44                   supported_sanitizers=['address', 'memory', 'undefined'],
45                   supported_architectures=['x86_64']),
46    'dataflow':
47        EngineInfo(upload_bucket='clusterfuzz-builds-dataflow',
48                   supported_sanitizers=['dataflow'],
49                   supported_architectures=['x86_64']),
50    'none':
51        EngineInfo(upload_bucket='clusterfuzz-builds-no-engine',
52                   supported_sanitizers=['address'],
53                   supported_architectures=['x86_64']),
54}
55
56
57def get_targets_list_filename(sanitizer):
58  return TARGETS_LIST_BASENAME + '.' + sanitizer
59
60
61def get_targets_list_url(bucket, project, sanitizer):
62  filename = get_targets_list_filename(sanitizer)
63  url = GCS_UPLOAD_URL_FORMAT.format(bucket, project, filename)
64  return url
65
66
67def _get_targets_list(project_name):
68  # libFuzzer ASan is the default configuration, get list of targets from it.
69  url = get_targets_list_url(ENGINE_INFO['libfuzzer'].upload_bucket,
70                             project_name, 'address')
71
72  url = urlparse.urljoin(GCS_URL_BASENAME, url)
73  response = requests.get(url)
74  if not response.status_code == 200:
75    sys.stderr.write('Failed to get list of targets from "%s".\n' % url)
76    sys.stderr.write('Status code: %d \t\tText:\n%s\n' %
77                     (response.status_code, response.text))
78    return None
79
80  return response.text.split()
81
82
83def get_signed_url(path, method='PUT', content_type=''):
84  timestamp = int(time.time() + BUILD_TIMEOUT)
85  blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path)
86
87  creds = ServiceAccountCredentials.from_json_keyfile_name(
88      os.environ['GOOGLE_APPLICATION_CREDENTIALS'])
89  client_id = creds.service_account_email
90  signature = base64.b64encode(creds.sign_blob(blob)[1])
91  values = {
92      'GoogleAccessId': client_id,
93      'Expires': timestamp,
94      'Signature': signature,
95  }
96
97  return ('https://storage.googleapis.com{0}?'.format(path) +
98          urllib.urlencode(values))
99
100
101def download_corpora_step(project_name):
102  """Returns a GCB step for downloading corpora backups for the given project.
103  """
104  fuzz_targets = _get_targets_list(project_name)
105  if not fuzz_targets:
106    sys.stderr.write('No fuzz targets found for project "%s".\n' % project_name)
107    return None
108
109  # Split fuzz targets into batches of CORPUS_DOWNLOAD_BATCH_SIZE.
110  for i in range(0, len(fuzz_targets), CORPUS_DOWNLOAD_BATCH_SIZE):
111    download_corpus_args = []
112    for binary_name in fuzz_targets[i:i + CORPUS_DOWNLOAD_BATCH_SIZE]:
113      qualified_name = binary_name
114      qualified_name_prefix = '%s_' % project_name
115      if not binary_name.startswith(qualified_name_prefix):
116        qualified_name = qualified_name_prefix + binary_name
117
118      url = get_signed_url(CORPUS_BACKUP_URL.format(project=project_name,
119                                                    fuzzer=qualified_name),
120                           method='GET')
121
122      corpus_archive_path = os.path.join('/corpus', binary_name + '.zip')
123      download_corpus_args.append('%s %s' % (corpus_archive_path, url))
124
125    step = {
126        'name': 'gcr.io/oss-fuzz-base/base-runner',
127        'entrypoint': 'download_corpus',
128        'args': download_corpus_args,
129        'volumes': [{
130            'name': 'corpus',
131            'path': '/corpus'
132        }],
133    }
134    return step
135