1# Copyright 2020 Google Inc. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# 15################################################################################ 16"""Utility module for Google Cloud Build scripts.""" 17import base64 18import collections 19import os 20import six.moves.urllib.parse as urlparse 21import sys 22import time 23 24import requests 25 26import google.auth 27import googleapiclient.discovery 28from oauth2client.service_account import ServiceAccountCredentials 29 30BUILD_TIMEOUT = 12 * 60 * 60 31 32# Needed for reading public target.list.* files. 33GCS_URL_BASENAME = 'https://storage.googleapis.com/' 34 35GCS_UPLOAD_URL_FORMAT = '/{0}/{1}/{2}' 36 37# Where corpus backups can be downloaded from. 38CORPUS_BACKUP_URL = ('/{project}-backup.clusterfuzz-external.appspot.com/' 39 'corpus/libFuzzer/{fuzzer}/latest.zip') 40 41# Cloud Builder has a limit of 100 build steps and 100 arguments for each step. 42CORPUS_DOWNLOAD_BATCH_SIZE = 100 43 44TARGETS_LIST_BASENAME = 'targets.list' 45 46EngineInfo = collections.namedtuple( 47 'EngineInfo', 48 ['upload_bucket', 'supported_sanitizers', 'supported_architectures']) 49 50ENGINE_INFO = { 51 'libfuzzer': 52 EngineInfo(upload_bucket='clusterfuzz-builds', 53 supported_sanitizers=['address', 'memory', 'undefined'], 54 supported_architectures=['x86_64', 'i386']), 55 'afl': 56 EngineInfo(upload_bucket='clusterfuzz-builds-afl', 57 supported_sanitizers=['address'], 58 supported_architectures=['x86_64']), 59 'honggfuzz': 60 EngineInfo(upload_bucket='clusterfuzz-builds-honggfuzz', 61 supported_sanitizers=['address'], 62 supported_architectures=['x86_64']), 63 'dataflow': 64 EngineInfo(upload_bucket='clusterfuzz-builds-dataflow', 65 supported_sanitizers=['dataflow'], 66 supported_architectures=['x86_64']), 67 'none': 68 EngineInfo(upload_bucket='clusterfuzz-builds-no-engine', 69 supported_sanitizers=['address'], 70 supported_architectures=['x86_64']), 71} 72 73 74def get_targets_list_filename(sanitizer): 75 """Returns target list filename.""" 76 return TARGETS_LIST_BASENAME + '.' + sanitizer 77 78 79def get_targets_list_url(bucket, project, sanitizer): 80 """Returns target list url.""" 81 filename = get_targets_list_filename(sanitizer) 82 url = GCS_UPLOAD_URL_FORMAT.format(bucket, project, filename) 83 return url 84 85 86def _get_targets_list(project_name): 87 """Returns target list.""" 88 # libFuzzer ASan is the default configuration, get list of targets from it. 89 url = get_targets_list_url(ENGINE_INFO['libfuzzer'].upload_bucket, 90 project_name, 'address') 91 92 url = urlparse.urljoin(GCS_URL_BASENAME, url) 93 response = requests.get(url) 94 if not response.status_code == 200: 95 sys.stderr.write('Failed to get list of targets from "%s".\n' % url) 96 sys.stderr.write('Status code: %d \t\tText:\n%s\n' % 97 (response.status_code, response.text)) 98 return None 99 100 return response.text.split() 101 102 103# pylint: disable=no-member 104def get_signed_url(path, method='PUT', content_type=''): 105 """Returns signed url.""" 106 timestamp = int(time.time() + BUILD_TIMEOUT) 107 blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path) 108 109 service_account_path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') 110 if service_account_path: 111 creds = ServiceAccountCredentials.from_json_keyfile_name( 112 os.environ['GOOGLE_APPLICATION_CREDENTIALS']) 113 client_id = creds.service_account_email 114 signature = base64.b64encode(creds.sign_blob(blob)[1]) 115 else: 116 credentials, project = google.auth.default() 117 iam = googleapiclient.discovery.build('iamcredentials', 118 'v1', 119 credentials=credentials, 120 cache_discovery=False) 121 client_id = project + '@appspot.gserviceaccount.com' 122 service_account = 'projects/-/serviceAccounts/{0}'.format(client_id) 123 response = iam.projects().serviceAccounts().signBlob( 124 name=service_account, 125 body={ 126 'delegates': [], 127 'payload': base64.b64encode(blob.encode('utf-8')).decode('utf-8'), 128 }).execute() 129 signature = response['signedBlob'] 130 131 values = { 132 'GoogleAccessId': client_id, 133 'Expires': timestamp, 134 'Signature': signature, 135 } 136 return ('https://storage.googleapis.com{0}?'.format(path) + 137 urlparse.urlencode(values)) 138 139 140def download_corpora_steps(project_name): 141 """Returns GCB steps for downloading corpora backups for the given project. 142 """ 143 fuzz_targets = _get_targets_list(project_name) 144 if not fuzz_targets: 145 sys.stderr.write('No fuzz targets found for project "%s".\n' % project_name) 146 return None 147 148 steps = [] 149 # Split fuzz targets into batches of CORPUS_DOWNLOAD_BATCH_SIZE. 150 for i in range(0, len(fuzz_targets), CORPUS_DOWNLOAD_BATCH_SIZE): 151 download_corpus_args = [] 152 for binary_name in fuzz_targets[i:i + CORPUS_DOWNLOAD_BATCH_SIZE]: 153 qualified_name = binary_name 154 qualified_name_prefix = '%s_' % project_name 155 if not binary_name.startswith(qualified_name_prefix): 156 qualified_name = qualified_name_prefix + binary_name 157 158 url = get_signed_url(CORPUS_BACKUP_URL.format(project=project_name, 159 fuzzer=qualified_name), 160 method='GET') 161 162 corpus_archive_path = os.path.join('/corpus', binary_name + '.zip') 163 download_corpus_args.append('%s %s' % (corpus_archive_path, url)) 164 165 steps.append({ 166 'name': 'gcr.io/oss-fuzz-base/base-runner', 167 'entrypoint': 'download_corpus', 168 'args': download_corpus_args, 169 'volumes': [{ 170 'name': 'corpus', 171 'path': '/corpus' 172 }], 173 }) 174 175 return steps 176 177 178def http_upload_step(data, signed_url, content_type): 179 """Returns a GCB step to upload data to the given URL via GCS HTTP API.""" 180 step = { 181 'name': 182 'gcr.io/cloud-builders/curl', 183 'args': [ 184 '-H', 185 'Content-Type: ' + content_type, 186 '-X', 187 'PUT', 188 '-d', 189 data, 190 signed_url, 191 ], 192 } 193 return step 194 195 196def gsutil_rm_rf_step(url): 197 """Returns a GCB step to recursively delete the object with given GCS url.""" 198 step = { 199 'name': 'gcr.io/cloud-builders/gsutil', 200 'entrypoint': 'sh', 201 'args': [ 202 '-c', 203 'gsutil -m rm -rf %s || exit 0' % url, 204 ], 205 } 206 return step 207 208 209def project_image_steps(name, image, language): 210 """Returns GCB steps to build OSS-Fuzz project image.""" 211 steps = [{ 212 'args': [ 213 'clone', 214 'https://github.com/google/oss-fuzz.git', 215 ], 216 'name': 'gcr.io/cloud-builders/git', 217 }, { 218 'name': 'gcr.io/cloud-builders/docker', 219 'args': [ 220 'build', 221 '-t', 222 image, 223 '.', 224 ], 225 'dir': 'oss-fuzz/projects/' + name, 226 }, { 227 'name': 228 image, 229 'args': [ 230 'bash', '-c', 231 'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json' 232 ], 233 'env': [ 234 'OSSFUZZ_REVISION=$REVISION_ID', 235 'FUZZING_LANGUAGE=%s' % language, 236 ], 237 }] 238 239 return steps 240