• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python2
2"""Starts project build on Google Cloud Builder.
3
4Usage: build_project.py <project_dir>
5"""
6
7from __future__ import print_function
8
9import datetime
10import json
11import os
12import re
13import sys
14import yaml
15
16from oauth2client.client import GoogleCredentials
17from googleapiclient.discovery import build
18
19import build_lib
20
21FUZZING_BUILD_TAG = 'fuzzing'
22
23GCB_LOGS_BUCKET = 'oss-fuzz-gcb-logs'
24
25CONFIGURATIONS = {
26    'sanitizer-address': ['SANITIZER=address'],
27    'sanitizer-dataflow': ['SANITIZER=dataflow'],
28    'sanitizer-memory': ['SANITIZER=memory'],
29    'sanitizer-undefined': ['SANITIZER=undefined'],
30    'engine-libfuzzer': ['FUZZING_ENGINE=libfuzzer'],
31    'engine-afl': ['FUZZING_ENGINE=afl'],
32    'engine-honggfuzz': ['FUZZING_ENGINE=honggfuzz'],
33    'engine-dataflow': ['FUZZING_ENGINE=dataflow'],
34    'engine-none': ['FUZZING_ENGINE=none'],
35}
36
37DEFAULT_ARCHITECTURES = ['x86_64']
38DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz']
39DEFAULT_SANITIZERS = ['address', 'undefined']
40
41
42def usage():
43  sys.stderr.write('Usage: ' + sys.argv[0] + ' <project_dir>\n')
44  exit(1)
45
46
47def load_project_yaml(project_dir):
48  project_name = os.path.basename(project_dir)
49  project_yaml_path = os.path.join(project_dir, 'project.yaml')
50  with open(project_yaml_path) as f:
51    project_yaml = yaml.safe_load(f)
52    project_yaml.setdefault('disabled', False)
53    project_yaml.setdefault('name', project_name)
54    project_yaml.setdefault('image', 'gcr.io/oss-fuzz/' + project_name)
55    project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES)
56    project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
57    project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
58    project_yaml.setdefault('run_tests', True)
59    project_yaml.setdefault('coverage_extra_args', '')
60    project_yaml.setdefault('labels', {})
61    project_yaml.setdefault('language', 'cpp')
62    return project_yaml
63
64
65def is_supported_configuration(fuzzing_engine, sanitizer, architecture):
66  fuzzing_engine_info = build_lib.ENGINE_INFO[fuzzing_engine]
67  if architecture == 'i386' and sanitizer != 'address':
68    return False
69  return (sanitizer in fuzzing_engine_info.supported_sanitizers and
70          architecture in fuzzing_engine_info.supported_architectures)
71
72
73def get_sanitizers(project_yaml):
74  sanitizers = project_yaml['sanitizers']
75  assert isinstance(sanitizers, list)
76
77  processed_sanitizers = []
78  for sanitizer in sanitizers:
79    if isinstance(sanitizer, basestring):
80      processed_sanitizers.append(sanitizer)
81    elif isinstance(sanitizer, dict):
82      for key in sanitizer.iterkeys():
83        processed_sanitizers.append(key)
84
85  return processed_sanitizers
86
87
88def workdir_from_dockerfile(dockerfile):
89  """Parse WORKDIR from the Dockerfile."""
90  WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
91
92  with open(dockerfile) as f:
93    lines = f.readlines()
94
95  for line in lines:
96    match = re.match(WORKDIR_REGEX, line)
97    if match:
98      # We need to escape '$' since they're used for subsitutions in Container
99      # Builer builds.
100      return match.group(1).replace('$', '$$')
101
102  return None
103
104
105def get_build_steps(project_dir):
106  project_yaml = load_project_yaml(project_dir)
107  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
108  name = project_yaml['name']
109  image = project_yaml['image']
110  run_tests = project_yaml['run_tests']
111
112  ts = datetime.datetime.now().strftime('%Y%m%d%H%M')
113
114  build_steps = [
115      {
116          'args': [
117              'clone',
118              'https://github.com/google/oss-fuzz.git',
119          ],
120          'name': 'gcr.io/cloud-builders/git',
121      },
122      {
123          'name': 'gcr.io/cloud-builders/docker',
124          'args': [
125              'build',
126              '-t',
127              image,
128              '.',
129          ],
130          'dir': 'oss-fuzz/projects/' + name,
131      },
132      {
133          'name': image,
134          'args': [
135              'bash', '-c',
136              'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
137          ],
138          'env': ['OSSFUZZ_REVISION=$REVISION_ID'],
139      },
140      {
141          'name': 'gcr.io/oss-fuzz-base/msan-builder',
142          'args': [
143              'bash',
144              '-c',
145              'cp -r /msan /workspace',
146          ],
147      },
148  ]
149
150  for fuzzing_engine in project_yaml['fuzzing_engines']:
151    for sanitizer in get_sanitizers(project_yaml):
152      for architecture in project_yaml['architectures']:
153        if not is_supported_configuration(fuzzing_engine, sanitizer,
154                                          architecture):
155          continue
156
157        env = CONFIGURATIONS['engine-' + fuzzing_engine][:]
158        env.extend(CONFIGURATIONS['sanitizer-' + sanitizer])
159        out = '/workspace/out/' + sanitizer
160        stamped_name = '-'.join([name, sanitizer, ts])
161        zip_file = stamped_name + '.zip'
162        stamped_srcmap_file = stamped_name + '.srcmap.json'
163        bucket = build_lib.ENGINE_INFO[fuzzing_engine].upload_bucket
164        if architecture != 'x86_64':
165          bucket += '-' + architecture
166        upload_url = build_lib.get_signed_url(
167            build_lib.GCS_UPLOAD_URL_FORMAT.format(bucket, name, zip_file))
168        srcmap_url = build_lib.get_signed_url(
169            build_lib.GCS_UPLOAD_URL_FORMAT.format(bucket, name,
170                                                   stamped_srcmap_file))
171
172        targets_list_filename = build_lib.get_targets_list_filename(sanitizer)
173        targets_list_url = build_lib.get_signed_url(
174            build_lib.get_targets_list_url(bucket, name, sanitizer))
175
176        env.append('OUT=' + out)
177        env.append('MSAN_LIBS_PATH=/workspace/msan')
178        env.append('ARCHITECTURE=' + architecture)
179
180        workdir = workdir_from_dockerfile(dockerfile_path)
181        if not workdir:
182          workdir = '/src'
183
184        failure_msg = ('*' * 80 + '\nFailed to build.\nTo reproduce, run:\n'
185                       'python infra/helper.py build_image {name}\n'
186                       'python infra/helper.py build_fuzzers --sanitizer '
187                       '{sanitizer} --engine {engine} --architecture '
188                       '{architecture} {name}\n' + '*' * 80).format(
189                           name=name,
190                           sanitizer=sanitizer,
191                           engine=fuzzing_engine,
192                           architecture=architecture)
193
194        build_steps.append(
195            # compile
196            {
197                'name':
198                    image,
199                'env':
200                    env,
201                'args': [
202                    'bash',
203                    '-c',
204                    # Remove /out to break loudly when a build script
205                    # incorrectly uses /out instead of $OUT.
206                    # `cd /src && cd {workdir}` (where {workdir} is parsed from
207                    # the Dockerfile). Container Builder overrides our workdir
208                    # so we need to add this step to set it back.
209                    ('rm -r /out && cd /src && cd {workdir} && mkdir -p {out} && '
210                     'compile || (echo "{failure_msg}" && false)'
211                    ).format(workdir=workdir, out=out, failure_msg=failure_msg),
212                ],
213            })
214
215        if sanitizer == 'memory':
216          # Patch dynamic libraries to use instrumented ones.
217          build_steps.append({
218              'name':
219                  'gcr.io/oss-fuzz-base/msan-builder',
220              'args': [
221                  'bash',
222                  '-c',
223                  # TODO(ochang): Replace with just patch_build.py once
224                  # permission in image is fixed.
225                  'python /usr/local/bin/patch_build.py {0}'.format(out),
226              ],
227          })
228
229        if run_tests:
230          failure_msg = ('*' * 80 + '\nBuild checks failed.\n'
231                         'To reproduce, run:\n'
232                         'python infra/helper.py build_image {name}\n'
233                         'python infra/helper.py build_fuzzers --sanitizer '
234                         '{sanitizer} --engine {engine} --architecture '
235                         '{architecture} {name}\n'
236                         'python infra/helper.py check_build --sanitizer '
237                         '{sanitizer} --engine {engine} --architecture '
238                         '{architecture} {name}\n' + '*' * 80).format(
239                             name=name,
240                             sanitizer=sanitizer,
241                             engine=fuzzing_engine,
242                             architecture=architecture)
243
244          build_steps.append(
245              # test binaries
246              {
247                  'name':
248                      'gcr.io/oss-fuzz-base/base-runner',
249                  'env':
250                      env,
251                  'args': [
252                      'bash', '-c',
253                      'test_all || (echo "{0}" && false)'.format(failure_msg)
254                  ],
255              })
256
257        if project_yaml['labels']:
258          # write target labels
259          build_steps.append({
260              'name':
261                  image,
262              'env':
263                  env,
264              'args': [
265                  '/usr/local/bin/write_labels.py',
266                  json.dumps(project_yaml['labels']),
267                  out,
268              ],
269          })
270
271        if sanitizer == 'dataflow' and fuzzing_engine == 'dataflow':
272          dataflow_steps = dataflow_post_build_steps(name, env)
273          if dataflow_steps:
274            build_steps.extend(dataflow_steps)
275          else:
276            sys.stderr.write('Skipping dataflow post build steps.\n')
277
278        build_steps.extend([
279            # generate targets list
280            {
281                'name':
282                    'gcr.io/oss-fuzz-base/base-runner',
283                'env':
284                    env,
285                'args': [
286                    'bash',
287                    '-c',
288                    'targets_list > /workspace/{0}'.format(
289                        targets_list_filename),
290                ],
291            },
292            # zip binaries
293            {
294                'name':
295                    image,
296                'args': [
297                    'bash', '-c',
298                    'cd {out} && zip -r {zip_file} *'.format(out=out,
299                                                             zip_file=zip_file)
300                ],
301            },
302            # upload srcmap
303            {
304                'name': 'gcr.io/oss-fuzz-base/uploader',
305                'args': [
306                    '/workspace/srcmap.json',
307                    srcmap_url,
308                ],
309            },
310            # upload binaries
311            {
312                'name': 'gcr.io/oss-fuzz-base/uploader',
313                'args': [
314                    os.path.join(out, zip_file),
315                    upload_url,
316                ],
317            },
318            # upload targets list
319            {
320                'name':
321                    'gcr.io/oss-fuzz-base/uploader',
322                'args': [
323                    '/workspace/{0}'.format(targets_list_filename),
324                    targets_list_url,
325                ],
326            },
327            # cleanup
328            {
329                'name': image,
330                'args': [
331                    'bash',
332                    '-c',
333                    'rm -r ' + out,
334                ],
335            },
336        ])
337
338  return build_steps
339
340
341def dataflow_post_build_steps(project_name, env):
342  steps = []
343  download_corpora_step = build_lib.download_corpora_step(project_name)
344  if not download_corpora_step:
345    return None
346
347  steps = [download_corpora_step]
348  steps.append({
349      'name': 'gcr.io/oss-fuzz-base/base-runner',
350      'env': env,
351      'args': [
352          'bash', '-c',
353          ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*}; done && '
354           'collect_dft || (echo "DFT collection failed." && false)')
355      ],
356      'volumes': [{
357          'name': 'corpus',
358          'path': '/corpus'
359      }],
360  })
361  return steps
362
363
364def get_logs_url(build_id):
365  URL_FORMAT = ('https://console.developers.google.com/logs/viewer?'
366                'resource=build%2Fbuild_id%2F{0}&project=oss-fuzz')
367  return URL_FORMAT.format(build_id)
368
369
370def run_build(build_steps, project_name, tag):
371  options = {}
372  if 'GCB_OPTIONS' in os.environ:
373    options = yaml.safe_load(os.environ['GCB_OPTIONS'])
374
375  build_body = {
376      'steps': build_steps,
377      'timeout': str(build_lib.BUILD_TIMEOUT) + 's',
378      'options': options,
379      'logsBucket': GCB_LOGS_BUCKET,
380      'tags': [project_name + '-' + tag,],
381  }
382
383  credentials = GoogleCredentials.get_application_default()
384  cloudbuild = build('cloudbuild', 'v1', credentials=credentials)
385  build_info = cloudbuild.projects().builds().create(projectId='oss-fuzz',
386                                                     body=build_body).execute()
387  build_id = build_info['metadata']['build']['id']
388
389  print('Logs:', get_logs_url(build_id), file=sys.stderr)
390  print(build_id)
391
392
393def main():
394  if len(sys.argv) != 2:
395    usage()
396
397  project_dir = sys.argv[1].rstrip(os.path.sep)
398  steps = get_build_steps(project_dir)
399
400  project_name = os.path.basename(project_dir)
401  run_build(steps, project_name, FUZZING_BUILD_TAG)
402
403
404if __name__ == '__main__':
405  main()
406