• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import sys
25import shutil
26import tarfile
27import zipfile
28import hashlib
29import argparse
30import subprocess
31import ssl
32import stat
33import platform
34import logging
35
36scriptPath = os.path.join(os.path.dirname(__file__), "..", "scripts")
37sys.path.insert(0, scriptPath)
38
39from ctsbuild.common import *
40
41EXTERNAL_DIR = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
42
43SYSTEM_NAME = platform.system()
44
45def computeChecksum (data):
46    return hashlib.sha256(data).hexdigest()
47
48def onReadonlyRemoveError (func, path, exc_info):
49    os.chmod(path, stat.S_IWRITE)
50    os.unlink(path)
51
52class Source:
53    def __init__(self, baseDir, extractDir):
54        self.baseDir = baseDir
55        self.extractDir = extractDir
56
57    def clean (self):
58        fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
59        # Remove read-only first
60        readonlydir = os.path.join(fullDstPath, ".git")
61        if os.path.exists(readonlydir):
62            logging.debug("Deleting " + readonlydir)
63            shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
64        if os.path.exists(fullDstPath):
65            logging.debug("Deleting " + fullDstPath)
66            shutil.rmtree(fullDstPath, ignore_errors=False)
67
68class SourcePackage (Source):
69    def __init__(self, url, checksum, baseDir, extractDir = "src", postExtract=None):
70        Source.__init__(self, baseDir, extractDir)
71        self.url = url
72        self.filename = os.path.basename(self.url)
73        self.checksum = checksum
74        self.archiveDir = "packages"
75        self.postExtract = postExtract
76
77    def clean (self):
78        Source.clean(self)
79        self.removeArchives()
80
81    def update (self, cmdProtocol = None, force = False):
82        if not self.isArchiveUpToDate():
83            self.fetchAndVerifyArchive()
84
85        if self.getExtractedChecksum() != self.checksum:
86            Source.clean(self)
87            self.extract()
88            self.storeExtractedChecksum(self.checksum)
89
90    def removeArchives (self):
91        archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
92        if os.path.exists(archiveDir):
93            logging.debug("Deleting " + archiveDir)
94            shutil.rmtree(archiveDir, ignore_errors=False)
95
96    def isArchiveUpToDate (self):
97        archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
98        if os.path.exists(archiveFile):
99            return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
100        else:
101            return False
102
103    def getExtractedChecksumFilePath (self):
104        return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
105
106    def getExtractedChecksum (self):
107        extractedChecksumFile = self.getExtractedChecksumFilePath()
108
109        if os.path.exists(extractedChecksumFile):
110            return readFile(extractedChecksumFile)
111        else:
112            return None
113
114    def storeExtractedChecksum (self, checksum):
115        checksum_bytes = checksum.encode("utf-8")
116        writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
117
118    def connectToUrl (self, url):
119        result = None
120
121        if sys.version_info < (3, 0):
122            from urllib2 import urlopen
123        else:
124            from urllib.request import urlopen
125
126        if args.insecure:
127            print("Ignoring certificate checks")
128            ssl_context = ssl._create_unverified_context()
129            result = urlopen(url, context=ssl_context)
130        else:
131            result = urlopen(url)
132
133        return result
134
135    def fetchAndVerifyArchive (self):
136        print("Fetching %s" % self.url)
137
138        req = self.connectToUrl(self.url)
139        data = req.read()
140        checksum = computeChecksum(data)
141        dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
142
143        if checksum != self.checksum:
144            raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
145
146        if not os.path.exists(os.path.dirname(dstPath)):
147            os.makedirs(os.path.dirname(dstPath))
148
149        writeBinaryFile(dstPath, data)
150
151    def extract (self):
152        print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
153
154        srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
155        tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
156        dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
157
158        if self.filename.endswith(".zip"):
159            archive = zipfile.ZipFile(srcPath)
160        else:
161            archive = tarfile.open(srcPath)
162
163        if os.path.exists(tmpPath):
164            shutil.rmtree(tmpPath, ignore_errors=False)
165
166        os.mkdir(tmpPath)
167
168        archive.extractall(tmpPath)
169        archive.close()
170
171        extractedEntries = os.listdir(tmpPath)
172        if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
173            raise Exception("%s doesn't contain single top-level directory" % self.filename)
174
175        topLevelPath = os.path.join(tmpPath, extractedEntries[0])
176
177        if not os.path.exists(dstPath):
178            os.mkdir(dstPath)
179
180        for entry in os.listdir(topLevelPath):
181            if os.path.exists(os.path.join(dstPath, entry)):
182                raise Exception("%s exists already" % entry)
183
184            shutil.move(os.path.join(topLevelPath, entry), dstPath)
185
186        shutil.rmtree(tmpPath, ignore_errors=True)
187
188        if self.postExtract != None:
189            self.postExtract(dstPath)
190
191class SourceFile (Source):
192    def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
193        Source.__init__(self, baseDir, extractDir)
194        self.url = url
195        self.filename = filename
196        self.checksum = checksum
197
198    def update (self, cmdProtocol = None, force = False):
199        if not self.isFileUpToDate():
200            Source.clean(self)
201            self.fetchAndVerifyFile()
202
203    def isFileUpToDate (self):
204        file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
205        if os.path.exists(file):
206            data = readFile(file)
207            return computeChecksum(data.encode('utf-8')) == self.checksum
208        else:
209            return False
210
211    def connectToUrl (self, url):
212        result = None
213
214        if sys.version_info < (3, 0):
215            from urllib2 import urlopen
216        else:
217            from urllib.request import urlopen
218
219        if args.insecure:
220            print("Ignoring certificate checks")
221            ssl_context = ssl._create_unverified_context()
222            result = urlopen(url, context=ssl_context)
223        else:
224            result = urlopen(url)
225
226        return result
227
228    def fetchAndVerifyFile (self):
229        print("Fetching %s" % self.url)
230
231        req = self.connectToUrl(self.url)
232        data = req.read()
233        checksum = computeChecksum(data)
234        dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
235
236        if checksum != self.checksum:
237            raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
238
239        if not os.path.exists(os.path.dirname(dstPath)):
240            os.mkdir(os.path.dirname(dstPath))
241
242        writeBinaryFile(dstPath, data)
243
244class GitRepo (Source):
245    def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
246        Source.__init__(self, baseDir, extractDir)
247        self.httpsUrl = httpsUrl
248        self.sshUrl = sshUrl
249        self.revision = revision
250        self.removeTags = removeTags
251        self.patch = patch
252
253    def checkout(self, url, fullDstPath, force):
254        print("Directory: " + fullDstPath)
255        for tag in self.removeTags:
256            proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
257            (stdout, stderr) = proc.communicate()
258            if len(stdout) > 0:
259                run(["git", "tag", "-d",tag])
260
261        force_arg = ['--force'] if force else []
262        try:
263            run(["git", "checkout"] + force_arg + [self.revision])
264        except KeyboardInterrupt:
265            # Propagate the exception to stop the process if possible.
266            raise
267        except:
268            logging.debug("couldn't find revision " + self.revision + " locally; fetching")
269            run(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
270            run(["git", "checkout"] + force_arg + [self.revision])
271
272        if(self.patch != ""):
273            patchFile = os.path.join(EXTERNAL_DIR, self.patch)
274            run(["git", "reset", "--hard", "HEAD"])
275            run(["git", "apply", patchFile])
276
277    def update (self, cmdProtocol, force = False):
278        fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
279        url         = self.httpsUrl
280        backupUrl   = self.sshUrl
281
282        # If url is none then start with ssh
283        if cmdProtocol == 'ssh' or url == None:
284            url       = self.sshUrl
285            backupUrl = self.httpsUrl
286
287        if not os.path.exists(os.path.join(fullDstPath, '.git')):
288            logging.debug("git repository does not exist; performing full clone")
289            try:
290                run(["git", "clone", "--no-checkout", url, fullDstPath])
291            except:
292                if backupUrl != None:
293                    execute(["git", "clone", "--no-checkout", backupUrl, fullDstPath])
294
295        pushWorkingDir(fullDstPath)
296
297        try:
298            self.checkout(url, fullDstPath, force)
299        except KeyboardInterrupt:
300            # Propagate the exception to stop the process if possible.
301            raise
302        finally:
303            popWorkingDir()
304
305def postExtractLibpng (path):
306    shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
307                os.path.join(path, "pnglibconf.h"))
308
309PACKAGES = [
310    SourcePackage(
311        "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz",
312        "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
313        "zlib"),
314    SourcePackage(
315        "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
316        "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
317        "libpng",
318        postExtract = postExtractLibpng),
319    SourceFile(
320        "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
321        "renderdoc_app.h",
322        "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
323        "renderdoc"),
324    GitRepo(
325        "https://github.com/KhronosGroup/SPIRV-Tools.git",
326        "git@github.com:KhronosGroup/SPIRV-Tools.git",
327        "3fb52548bc8a68d349d31e21bd4e80e3d953e87c",
328        "spirv-tools"),
329    GitRepo(
330        "https://github.com/KhronosGroup/glslang.git",
331        "git@github.com:KhronosGroup/glslang.git",
332        "3a2834e7702651043ca9f35d022739e740563516",
333        "glslang",
334        removeTags = ["main-tot", "master-tot"]),
335    GitRepo(
336        "https://github.com/KhronosGroup/SPIRV-Headers.git",
337        "git@github.com:KhronosGroup/SPIRV-Headers.git",
338        "36d5e2ddaa54c70d2f29081510c66f4fc98e5e53",
339        "spirv-headers"),
340    GitRepo(
341        "https://github.com/KhronosGroup/Vulkan-Docs.git",
342        "git@github.com:KhronosGroup/Vulkan-Docs.git",
343        "c7a3955e47d223c6a37fb29e2061c973eec98d0a",
344        "vulkan-docs"),
345    GitRepo(
346        "https://github.com/KhronosGroup/Vulkan-ValidationLayers.git",
347        "git@github.com:KhronosGroup/Vulkan-ValidationLayers.git",
348        "902f3cf8d51e76be0c0deb4be39c6223abebbae2",
349        "vulkan-validationlayers"),
350    GitRepo(
351        "https://github.com/google/amber.git",
352        "git@github.com:google/amber.git",
353        "1ec5e96db7e0343d045a52c590e30eba154f74a8",
354        "amber"),
355    GitRepo(
356        "https://github.com/open-source-parsers/jsoncpp.git",
357        "git@github.com:open-source-parsers/jsoncpp.git",
358        "9059f5cad030ba11d37818847443a53918c327b1",
359        "jsoncpp"),
360    # NOTE: The samples application is not well suited to external
361    # integration, this fork contains the small fixes needed for use
362    # by the CTS.
363    GitRepo(
364        "https://github.com/Igalia/vk_video_samples.git",
365        "git@github.com:Igalia/vk_video_samples.git",
366        "45fe88b456c683120138f052ea81f0a958ff3ec4",
367        "nvidia-video-samples"),
368    # NOTE: Temporary vk_video_samples repo and branch where AV1
369    # encoder library is being developed by NVidia.
370    GitRepo(
371        "https://github.com/KhronosGroup/Vulkan-Video-Samples.git",
372        "git@github.com:KhronosGroup/Vulkan-Video-Samples.git",
373        "0e87744edbb84c9c56c3fc8de9ea5150af5ee4ea",
374        "vulkan-video-samples"),
375    # NOTE: Temporary video generator repo .
376    GitRepo(
377        "https://github.com/Igalia/video_generator.git",
378        "git@github.com:Igalia/video_generator.git",
379        "426300e12a5cc5d4676807039a1be237a2b68187",
380        "video_generator"),
381]
382
383def parseArgs ():
384    versionsForInsecure = ((2,7,9), (3,4,3))
385    versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
386
387    parser = argparse.ArgumentParser(description = "Fetch external sources")
388    parser.add_argument('--clean', dest='clean', action='store_true', default=False,
389                        help='Remove sources instead of fetching')
390    parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
391                        help="Disable certificate check for external sources."
392                        " Minimum python version required " + versionsForInsecureStr)
393    parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
394                        help="Select protocol to checkout git repositories.")
395    parser.add_argument('--force', dest='force', action='store_true', default=False,
396                        help="Pass --force to git fetch and checkout commands")
397    parser.add_argument("-v", "--verbose",
398                        dest="verbose",
399                        action="store_true",
400                        help="Enable verbose logging")
401    args = parser.parse_args()
402
403    if args.insecure:
404        for versionItem in versionsForInsecure:
405            if (sys.version_info.major == versionItem[0]):
406                if sys.version_info < versionItem:
407                    parser.error("For --insecure minimum required python version is " +
408                                versionsForInsecureStr)
409                break;
410
411    return args
412
413def run(*popenargs, **kwargs):
414    process = subprocess.Popen(*popenargs, **kwargs)
415
416    try:
417        stdout, stderr = process.communicate(None)
418    except KeyboardInterrupt:
419        # Terminate the process, wait and propagate.
420        process.terminate()
421        process.wait()
422        raise
423    except:
424        # With any other exception, we _kill_ the process and propagate.
425        process.kill()
426        process.wait()
427        raise
428    else:
429        # Everything good, fetch the retcode and raise exception if needed.
430        retcode = process.poll()
431        if retcode:
432            raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
433
434if __name__ == "__main__":
435    args = parseArgs()
436    initializeLogger(args.verbose)
437
438    try:
439        for pkg in PACKAGES:
440            if args.clean:
441                pkg.clean()
442            else:
443                pkg.update(args.protocol, args.force)
444    except KeyboardInterrupt:
445        sys.exit("") # Returns 1.
446