• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import sys
25import shutil
26import tarfile
27import zipfile
28import hashlib
29import argparse
30import subprocess
31import ssl
32import stat
33import platform
34
35sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
36
37from ctsbuild.common import *
38
39EXTERNAL_DIR	= os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
40
41SYSTEM_NAME		= platform.system()
42
43def computeChecksum (data):
44	return hashlib.sha256(data).hexdigest()
45
46def onReadonlyRemoveError (func, path, exc_info):
47	os.chmod(path, stat.S_IWRITE)
48	os.unlink(path)
49
50class Source:
51	def __init__(self, baseDir, extractDir):
52		self.baseDir		= baseDir
53		self.extractDir		= extractDir
54
55	def clean (self):
56		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
57		# Remove read-only first
58		readonlydir = os.path.join(fullDstPath, ".git")
59		if os.path.exists(readonlydir):
60			shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
61		if os.path.exists(fullDstPath):
62			shutil.rmtree(fullDstPath, ignore_errors=False)
63
64class SourcePackage (Source):
65	def __init__(self, url, checksum, baseDir, extractDir = "src", postExtract=None):
66		Source.__init__(self, baseDir, extractDir)
67		self.url			= url
68		self.filename		= os.path.basename(self.url)
69		self.checksum		= checksum
70		self.archiveDir		= "packages"
71		self.postExtract	= postExtract
72
73		if SYSTEM_NAME == 'Windows' or SYSTEM_NAME.startswith('CYGWIN') or SYSTEM_NAME.startswith('MINGW'):
74			self.sysNdx = 0
75		elif SYSTEM_NAME == 'Linux':
76			self.sysNdx = 1
77		elif SYSTEM_NAME == 'Darwin':
78			self.sysNdx = 2
79		else:
80			self.sysNdx = -1  # unknown system
81
82		self.FFmpeg			= "FFmpeg" in url
83
84	def clean (self):
85		Source.clean(self)
86		self.removeArchives()
87
88	def update (self, cmdProtocol = None, force = False):
89		if self.sysNdx != 2:
90			if not self.isArchiveUpToDate():
91				self.fetchAndVerifyArchive()
92
93			if self.getExtractedChecksum() != self.checksum:
94				Source.clean(self)
95				self.extract()
96				self.storeExtractedChecksum(self.checksum)
97
98	def removeArchives (self):
99		archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
100		if os.path.exists(archiveDir):
101			shutil.rmtree(archiveDir, ignore_errors=False)
102
103	def isArchiveUpToDate (self):
104		archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
105		if os.path.exists(archiveFile):
106			return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
107		else:
108			return False
109
110	def getExtractedChecksumFilePath (self):
111		return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
112
113	def getExtractedChecksum (self):
114		extractedChecksumFile = self.getExtractedChecksumFilePath()
115
116		if os.path.exists(extractedChecksumFile):
117			return readFile(extractedChecksumFile)
118		else:
119			return None
120
121	def storeExtractedChecksum (self, checksum):
122		checksum_bytes = checksum.encode("utf-8")
123		writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
124
125	def connectToUrl (self, url):
126		result = None
127
128		if sys.version_info < (3, 0):
129			from urllib2 import urlopen
130		else:
131			from urllib.request import urlopen
132
133		if args.insecure:
134			print("Ignoring certificate checks")
135			ssl_context = ssl._create_unverified_context()
136			result = urlopen(url, context=ssl_context)
137		else:
138			result = urlopen(url)
139
140		return result
141
142	def fetchAndVerifyArchive (self):
143		print("Fetching %s" % self.url)
144
145		req			= self.connectToUrl(self.url)
146		data		= req.read()
147		checksum	= computeChecksum(data)
148		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
149
150		if checksum != self.checksum:
151			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
152
153		if not os.path.exists(os.path.dirname(dstPath)):
154			os.mkdir(os.path.dirname(dstPath))
155
156		writeBinaryFile(dstPath, data)
157
158	def extract (self):
159		print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
160
161		srcPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
162		tmpPath	= os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
163		dstPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
164
165		if self.filename.endswith(".zip"):
166			archive	= zipfile.ZipFile(srcPath)
167		else:
168			archive	= tarfile.open(srcPath)
169
170		if os.path.exists(tmpPath):
171			shutil.rmtree(tmpPath, ignore_errors=False)
172
173		os.mkdir(tmpPath)
174
175		archive.extractall(tmpPath)
176		archive.close()
177
178		extractedEntries = os.listdir(tmpPath)
179		if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
180			raise Exception("%s doesn't contain single top-level directory" % self.filename)
181
182		topLevelPath = os.path.join(tmpPath, extractedEntries[0])
183
184		if not os.path.exists(dstPath):
185			os.mkdir(dstPath)
186
187		for entry in os.listdir(topLevelPath):
188			if os.path.exists(os.path.join(dstPath, entry)):
189				raise Exception("%s exists already" % entry)
190
191			shutil.move(os.path.join(topLevelPath, entry), dstPath)
192
193		shutil.rmtree(tmpPath, ignore_errors=True)
194
195		if self.postExtract != None:
196			self.postExtract(dstPath)
197
198class SourceFile (Source):
199	def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
200		Source.__init__(self, baseDir, extractDir)
201		self.url			= url
202		self.filename		= filename
203		self.checksum		= checksum
204
205	def update (self, cmdProtocol = None, force = False):
206		if not self.isFileUpToDate():
207			Source.clean(self)
208			self.fetchAndVerifyFile()
209
210	def isFileUpToDate (self):
211		file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
212		if os.path.exists(file):
213			data = readFile(file)
214			return computeChecksum(data.encode('utf-8')) == self.checksum
215		else:
216			return False
217
218	def connectToUrl (self, url):
219		result = None
220
221		if sys.version_info < (3, 0):
222			from urllib2 import urlopen
223		else:
224			from urllib.request import urlopen
225
226		if args.insecure:
227			print("Ignoring certificate checks")
228			ssl_context = ssl._create_unverified_context()
229			result = urlopen(url, context=ssl_context)
230		else:
231			result = urlopen(url)
232
233		return result
234
235	def fetchAndVerifyFile (self):
236		print("Fetching %s" % self.url)
237
238		req			= self.connectToUrl(self.url)
239		data		= req.read()
240		checksum	= computeChecksum(data)
241		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
242
243		if checksum != self.checksum:
244			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
245
246		if not os.path.exists(os.path.dirname(dstPath)):
247			os.mkdir(os.path.dirname(dstPath))
248
249		writeBinaryFile(dstPath, data)
250
251class GitRepo (Source):
252	def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
253		Source.__init__(self, baseDir, extractDir)
254		self.httpsUrl	= httpsUrl
255		self.sshUrl		= sshUrl
256		self.revision	= revision
257		self.removeTags	= removeTags
258		self.patch		= patch
259
260	def checkout(self, url, fullDstPath, force):
261		if not os.path.exists(os.path.join(fullDstPath, '.git')):
262			execute(["git", "clone", "--no-checkout", url, fullDstPath])
263
264		pushWorkingDir(fullDstPath)
265		try:
266			for tag in self.removeTags:
267				proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
268				(stdout, stderr) = proc.communicate()
269				if len(stdout) > 0:
270					execute(["git", "tag", "-d",tag])
271			force_arg = ['--force'] if force else []
272			execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
273			execute(["git", "checkout"] + force_arg + [self.revision])
274
275			if(self.patch != ""):
276				patchFile = os.path.join(EXTERNAL_DIR, self.patch)
277				execute(["git", "reset", "--hard", "HEAD"])
278				execute(["git", "apply", patchFile])
279		finally:
280			popWorkingDir()
281
282	def update (self, cmdProtocol, force = False):
283		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
284		url         = self.httpsUrl
285		backupUrl   = self.sshUrl
286
287		# If url is none then start with ssh
288		if cmdProtocol == 'ssh' or url == None:
289			url       = self.sshUrl
290			backupUrl = self.httpsUrl
291
292		try:
293			self.checkout(url, fullDstPath, force)
294		except:
295			if backupUrl != None:
296				self.checkout(backupUrl, fullDstPath, force)
297
298def postExtractLibpng (path):
299	shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
300				os.path.join(path, "pnglibconf.h"))
301
302if SYSTEM_NAME == 'Windows' or SYSTEM_NAME.startswith('CYGWIN') or SYSTEM_NAME.startswith('MINGW'):
303    ffmpeg_url = "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2022-05-31-12-34/ffmpeg-n4.4.2-1-g8e98dfc57f-win64-lgpl-shared-4.4.zip"
304    ffmpeg_hash_value = "670df8e9d2ddd5e761459b3538f64b8826566270ef1ed13bcbfc63e73aab3fd9"
305elif SYSTEM_NAME == 'Linux':
306    ffmpeg_url = "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2022-05-31-12-34/ffmpeg-n4.4.2-1-g8e98dfc57f-linux64-gpl-shared-4.4.tar.xz"
307    ffmpeg_hash_value = "817f8c93ff1ef7ede3dad15b20415d5e366bcd6848844d55046111fd3de827d0"
308elif SYSTEM_NAME == 'Darwin':
309    ffmpeg_url = ""
310    ffmpeg_hash_value = ""
311else:
312    ffmpeg_url = None
313    ffmpeg_hash_value = None
314
315PACKAGES = [
316	SourcePackage(
317		"http://zlib.net/fossils/zlib-1.2.13.tar.gz",
318		"b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
319		"zlib"),
320	SourcePackage(
321		"http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
322		"c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
323		"libpng",
324		postExtract = postExtractLibpng),
325	SourcePackage(
326		ffmpeg_url,
327		ffmpeg_hash_value,
328		"ffmpeg"),
329	SourceFile(
330		"https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
331		"renderdoc_app.h",
332		"e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
333		"renderdoc"),
334	GitRepo(
335		"https://github.com/KhronosGroup/SPIRV-Tools.git",
336		"git@github.com:KhronosGroup/SPIRV-Tools.git",
337		"f98473ceeb1d33700d01e20910433583e5256030",
338		"spirv-tools"),
339	GitRepo(
340		"https://github.com/KhronosGroup/glslang.git",
341		"git@github.com:KhronosGroup/glslang.git",
342		"77417d5c9e0a5d4c79ddd0285d530b45f7259f0d",
343		"glslang",
344		removeTags = ["master-tot"]),
345	GitRepo(
346		"https://github.com/KhronosGroup/SPIRV-Headers.git",
347		"git@github.com:KhronosGroup/SPIRV-Headers.git",
348		"87d5b782bec60822aa878941e6b13c0a9a954c9b",
349		"spirv-headers"),
350	GitRepo(
351		"https://github.com/KhronosGroup/Vulkan-Docs.git",
352		"git@github.com:KhronosGroup/Vulkan-Docs.git",
353		"9a2e576a052a1e65a5d41b593e693ff02745604b",
354		"vulkan-docs"),
355	GitRepo(
356		"https://github.com/google/amber.git",
357		"git@github.com:google/amber.git",
358		"8b145a6c89dcdb4ec28173339dd176fb7b6f43ed",
359		"amber"),
360	GitRepo(
361		"https://github.com/open-source-parsers/jsoncpp.git",
362		"git@github.com:open-source-parsers/jsoncpp.git",
363		"9059f5cad030ba11d37818847443a53918c327b1",
364		"jsoncpp"),
365	GitRepo(
366		"https://github.com/nvpro-samples/vk_video_samples.git",
367		None,
368		"7d68747d3524842afaf050c5e00a10f5b8c07904",
369		"video-parser"),
370]
371
372def parseArgs ():
373	versionsForInsecure = ((2,7,9), (3,4,3))
374	versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
375
376	parser = argparse.ArgumentParser(description = "Fetch external sources")
377	parser.add_argument('--clean', dest='clean', action='store_true', default=False,
378						help='Remove sources instead of fetching')
379	parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
380						help="Disable certificate check for external sources."
381						" Minimum python version required " + versionsForInsecureStr)
382	parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
383						help="Select protocol to checkout git repositories.")
384	parser.add_argument('--force', dest='force', action='store_true', default=False,
385						help="Pass --force to git fetch and checkout commands")
386
387	args = parser.parse_args()
388
389	if args.insecure:
390		for versionItem in versionsForInsecure:
391			if (sys.version_info.major == versionItem[0]):
392				if sys.version_info < versionItem:
393					parser.error("For --insecure minimum required python version is " +
394								versionsForInsecureStr)
395				break;
396
397	return args
398
399def run(*popenargs, **kwargs):
400	process = subprocess.Popen(*popenargs, **kwargs)
401
402	try:
403		stdout, stderr = process.communicate(None)
404	except:
405		process.kill()
406		process.wait()
407		raise
408
409	retcode = process.poll()
410
411	if retcode:
412		raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
413
414	return retcode, stdout, stderr
415
416if __name__ == "__main__":
417	# Rerun script with python3 as python2 does not have lzma (xz) decompression support
418	if sys.version_info < (3, 0):
419		if SYSTEM_NAME == 'Windows' or SYSTEM_NAME.startswith('CYGWIN') or SYSTEM_NAME.startswith('MINGW'):
420			cmd = ['py', '-3']
421		elif SYSTEM_NAME == 'Linux':
422			cmd = ['python3']
423		elif SYSTEM_NAME == 'Darwin':
424			cmd = ['python3']
425		else:
426			cmd = None  # unknown system
427
428		cmd = cmd + sys.argv
429		run(cmd)
430	else:
431		args = parseArgs()
432
433		for pkg in PACKAGES:
434			if args.clean:
435				pkg.clean()
436			else:
437				pkg.update(args.protocol, args.force)
438