1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2015 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23import os 24import sys 25import shutil 26import tarfile 27import hashlib 28import argparse 29import subprocess 30import ssl 31import stat 32 33sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) 34 35from build.common import * 36 37EXTERNAL_DIR = os.path.realpath(os.path.normpath(os.path.dirname(__file__))) 38 39def computeChecksum (data): 40 return hashlib.sha256(data).hexdigest() 41 42def onReadonlyRemoveError (func, path, exc_info): 43 os.chmod(path, stat.S_IWRITE) 44 os.unlink(path) 45 46class Source: 47 def __init__(self, baseDir, extractDir): 48 self.baseDir = baseDir 49 self.extractDir = extractDir 50 51 def clean (self): 52 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir) 53 # Remove read-only first 54 readonlydir = os.path.join(fullDstPath, ".git", "objects", "pack") 55 if os.path.exists(readonlydir): 56 shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError ) 57 if os.path.exists(fullDstPath): 58 shutil.rmtree(fullDstPath, ignore_errors=False) 59 60class SourcePackage (Source): 61 def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None): 62 Source.__init__(self, baseDir, extractDir) 63 self.url = url 64 self.filename = filename 65 self.checksum = checksum 66 self.archiveDir = "packages" 67 self.postExtract = postExtract 68 69 def clean (self): 70 Source.clean(self) 71 self.removeArchives() 72 73 def update (self, cmdProtocol = None, force = False): 74 if not self.isArchiveUpToDate(): 75 self.fetchAndVerifyArchive() 76 77 if self.getExtractedChecksum() != self.checksum: 78 Source.clean(self) 79 self.extract() 80 self.storeExtractedChecksum(self.checksum) 81 82 def removeArchives (self): 83 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir) 84 if os.path.exists(archiveDir): 85 shutil.rmtree(archiveDir, ignore_errors=False) 86 87 def isArchiveUpToDate (self): 88 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename) 89 if os.path.exists(archiveFile): 90 return computeChecksum(readBinaryFile(archiveFile)) == self.checksum 91 else: 92 return False 93 94 def getExtractedChecksumFilePath (self): 95 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted") 96 97 def getExtractedChecksum (self): 98 extractedChecksumFile = self.getExtractedChecksumFilePath() 99 100 if os.path.exists(extractedChecksumFile): 101 return readFile(extractedChecksumFile) 102 else: 103 return None 104 105 def storeExtractedChecksum (self, checksum): 106 checksum_bytes = checksum.encode("utf-8") 107 writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes) 108 109 def connectToUrl (self, url): 110 result = None 111 112 if sys.version_info < (3, 0): 113 from urllib2 import urlopen 114 else: 115 from urllib.request import urlopen 116 117 if args.insecure: 118 print("Ignoring certificate checks") 119 ssl_context = ssl._create_unverified_context() 120 result = urlopen(url, context=ssl_context) 121 else: 122 result = urlopen(url) 123 124 return result 125 126 def fetchAndVerifyArchive (self): 127 print("Fetching %s" % self.url) 128 129 req = self.connectToUrl(self.url) 130 data = req.read() 131 checksum = computeChecksum(data) 132 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename) 133 134 if checksum != self.checksum: 135 raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum)) 136 137 if not os.path.exists(os.path.dirname(dstPath)): 138 os.mkdir(os.path.dirname(dstPath)) 139 140 writeBinaryFile(dstPath, data) 141 142 def extract (self): 143 print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir)) 144 145 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename) 146 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir) 147 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir) 148 archive = tarfile.open(srcPath) 149 150 if os.path.exists(tmpPath): 151 shutil.rmtree(tmpPath, ignore_errors=False) 152 153 os.mkdir(tmpPath) 154 155 archive.extractall(tmpPath) 156 archive.close() 157 158 extractedEntries = os.listdir(tmpPath) 159 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])): 160 raise Exception("%s doesn't contain single top-level directory" % self.filename) 161 162 topLevelPath = os.path.join(tmpPath, extractedEntries[0]) 163 164 if not os.path.exists(dstPath): 165 os.mkdir(dstPath) 166 167 for entry in os.listdir(topLevelPath): 168 if os.path.exists(os.path.join(dstPath, entry)): 169 raise Exception("%s exists already" % entry) 170 171 shutil.move(os.path.join(topLevelPath, entry), dstPath) 172 173 shutil.rmtree(tmpPath, ignore_errors=True) 174 175 if self.postExtract != None: 176 self.postExtract(dstPath) 177 178class SourceFile (Source): 179 def __init__(self, url, filename, checksum, baseDir, extractDir = "src"): 180 Source.__init__(self, baseDir, extractDir) 181 self.url = url 182 self.filename = filename 183 self.checksum = checksum 184 185 def update (self, cmdProtocol = None, force = False): 186 if not self.isFileUpToDate(): 187 Source.clean(self) 188 self.fetchAndVerifyFile() 189 190 def isFileUpToDate (self): 191 file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename) 192 if os.path.exists(file): 193 data = readFile(file) 194 return computeChecksum(data.encode('utf-8')) == self.checksum 195 else: 196 return False 197 198 def connectToUrl (self, url): 199 result = None 200 201 if sys.version_info < (3, 0): 202 from urllib2 import urlopen 203 else: 204 from urllib.request import urlopen 205 206 if args.insecure: 207 print("Ignoring certificate checks") 208 ssl_context = ssl._create_unverified_context() 209 result = urlopen(url, context=ssl_context) 210 else: 211 result = urlopen(url) 212 213 return result 214 215 def fetchAndVerifyFile (self): 216 print("Fetching %s" % self.url) 217 218 req = self.connectToUrl(self.url) 219 data = req.read() 220 checksum = computeChecksum(data) 221 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename) 222 223 if checksum != self.checksum: 224 raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum)) 225 226 if not os.path.exists(os.path.dirname(dstPath)): 227 os.mkdir(os.path.dirname(dstPath)) 228 229 writeBinaryFile(dstPath, data) 230 231class GitRepo (Source): 232 def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = []): 233 Source.__init__(self, baseDir, extractDir) 234 self.httpsUrl = httpsUrl 235 self.sshUrl = sshUrl 236 self.revision = revision 237 self.removeTags = removeTags 238 239 def detectProtocol(self, cmdProtocol = None): 240 # reuse parent repo protocol 241 proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE, universal_newlines=True) 242 (stdout, stderr) = proc.communicate() 243 244 if proc.returncode != 0: 245 raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode) 246 if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'): 247 protocol = 'ssh' 248 else: 249 # remote 'origin' doesn't exist, assume 'https' as checkout protocol 250 protocol = 'https' 251 return protocol 252 253 def selectUrl(self, cmdProtocol = None): 254 try: 255 if cmdProtocol == None: 256 protocol = self.detectProtocol(cmdProtocol) 257 else: 258 protocol = cmdProtocol 259 except: 260 # fallback to https on any issues 261 protocol = 'https' 262 263 if protocol == 'ssh': 264 if self.sshUrl != None: 265 url = self.sshUrl 266 else: 267 assert self.httpsUrl != None 268 url = self.httpsUrl 269 else: 270 assert protocol == 'https' 271 url = self.httpsUrl 272 273 assert url != None 274 return url 275 276 def update (self, cmdProtocol = None, force = False): 277 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir) 278 279 url = self.selectUrl(cmdProtocol) 280 if not os.path.exists(os.path.join(fullDstPath, '.git')): 281 execute(["git", "clone", "--no-checkout", url, fullDstPath]) 282 283 pushWorkingDir(fullDstPath) 284 try: 285 for tag in self.removeTags: 286 proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE) 287 (stdout, stderr) = proc.communicate() 288 if proc.returncode == 0: 289 execute(["git", "tag", "-d",tag]) 290 force_arg = ['--force'] if force else [] 291 execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"]) 292 execute(["git", "checkout"] + force_arg + [self.revision]) 293 finally: 294 popWorkingDir() 295 296def postExtractLibpng (path): 297 shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"), 298 os.path.join(path, "pnglibconf.h")) 299 300PACKAGES = [ 301 SourcePackage( 302 "http://zlib.net/zlib-1.2.11.tar.gz", 303 "zlib-1.2.11.tar.gz", 304 "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", 305 "zlib"), 306 SourcePackage( 307 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz", 308 "libpng-1.6.27.tar.gz", 309 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4", 310 "libpng", 311 postExtract = postExtractLibpng), 312 SourceFile( 313 "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h", 314 "renderdoc_app.h", 315 "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e", 316 "renderdoc"), 317 GitRepo( 318 "https://github.com/KhronosGroup/SPIRV-Tools.git", 319 None, 320 "f11f7434815838bbad349124767b258ce7df41f0", 321 "spirv-tools"), 322 GitRepo( 323 "https://github.com/KhronosGroup/glslang.git", 324 None, 325 "5c4f421121c4d24aad23a507e630dc5dc6c92c7c", 326 "glslang", 327 removeTags = ["master-tot"]), 328 GitRepo( 329 "https://github.com/KhronosGroup/SPIRV-Headers.git", 330 None, 331 "faa570afbc91ac73d594d787486bcf8f2df1ace0", 332 "spirv-headers"), 333 GitRepo( 334 "https://github.com/google/amber.git", 335 None, 336 "dabae26164714abf951c6815a2b4513260f7c6a4", 337 "amber"), 338] 339 340def parseArgs (): 341 versionsForInsecure = ((2,7,9), (3,4,3)) 342 versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure) 343 344 parser = argparse.ArgumentParser(description = "Fetch external sources") 345 parser.add_argument('--clean', dest='clean', action='store_true', default=False, 346 help='Remove sources instead of fetching') 347 parser.add_argument('--insecure', dest='insecure', action='store_true', default=False, 348 help="Disable certificate check for external sources." 349 " Minimum python version required " + versionsForInsecureStr) 350 parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'], 351 help="Select protocol to checkout git repositories.") 352 parser.add_argument('--force', dest='force', action='store_true', default=False, 353 help="Pass --force to git fetch and checkout commands") 354 355 args = parser.parse_args() 356 357 if args.insecure: 358 for versionItem in versionsForInsecure: 359 if (sys.version_info.major == versionItem[0]): 360 if sys.version_info < versionItem: 361 parser.error("For --insecure minimum required python version is " + 362 versionsForInsecureStr) 363 break; 364 365 return args 366 367if __name__ == "__main__": 368 args = parseArgs() 369 370 for pkg in PACKAGES: 371 if args.clean: 372 pkg.clean() 373 else: 374 pkg.update(args.protocol, args.force) 375