• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import sys
25import shutil
26import tarfile
27import zipfile
28import hashlib
29import argparse
30import subprocess
31import ssl
32import stat
33
34sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
35
36from ctsbuild.common import *
37
38EXTERNAL_DIR	= os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
39
40def computeChecksum (data):
41	return hashlib.sha256(data).hexdigest()
42
43def onReadonlyRemoveError (func, path, exc_info):
44	os.chmod(path, stat.S_IWRITE)
45	os.unlink(path)
46
47class Source:
48	def __init__(self, baseDir, extractDir):
49		self.baseDir		= baseDir
50		self.extractDir		= extractDir
51
52	def clean (self):
53		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
54		# Remove read-only first
55		readonlydir = os.path.join(fullDstPath, ".git")
56		if os.path.exists(readonlydir):
57			shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
58		if os.path.exists(fullDstPath):
59			shutil.rmtree(fullDstPath, ignore_errors=False)
60
61class SourcePackage (Source):
62	def __init__(self, url, checksum, baseDir, extractDir = "src", postExtract=None):
63		Source.__init__(self, baseDir, extractDir)
64		self.url			= url
65		self.filename		= os.path.basename(self.url)
66		self.checksum		= checksum
67		self.archiveDir		= "packages"
68		self.postExtract	= postExtract
69		self.sysNdx			= {"Windows":0, "Linux":1, "Darwin":2}[platform.system()]
70		self.FFmpeg			= "FFmpeg" in url
71
72	def clean (self):
73		Source.clean(self)
74		self.removeArchives()
75
76	def update (self, cmdProtocol = None, force = False):
77		if self.sysNdx != 2:
78			if not self.isArchiveUpToDate():
79				self.fetchAndVerifyArchive()
80
81			if self.getExtractedChecksum() != self.checksum:
82				Source.clean(self)
83				self.extract()
84				self.storeExtractedChecksum(self.checksum)
85
86	def removeArchives (self):
87		archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
88		if os.path.exists(archiveDir):
89			shutil.rmtree(archiveDir, ignore_errors=False)
90
91	def isArchiveUpToDate (self):
92		archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
93		if os.path.exists(archiveFile):
94			return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
95		else:
96			return False
97
98	def getExtractedChecksumFilePath (self):
99		return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
100
101	def getExtractedChecksum (self):
102		extractedChecksumFile = self.getExtractedChecksumFilePath()
103
104		if os.path.exists(extractedChecksumFile):
105			return readFile(extractedChecksumFile)
106		else:
107			return None
108
109	def storeExtractedChecksum (self, checksum):
110		checksum_bytes = checksum.encode("utf-8")
111		writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
112
113	def connectToUrl (self, url):
114		result = None
115
116		if sys.version_info < (3, 0):
117			from urllib2 import urlopen
118		else:
119			from urllib.request import urlopen
120
121		if args.insecure:
122			print("Ignoring certificate checks")
123			ssl_context = ssl._create_unverified_context()
124			result = urlopen(url, context=ssl_context)
125		else:
126			result = urlopen(url)
127
128		return result
129
130	def fetchAndVerifyArchive (self):
131		print("Fetching %s" % self.url)
132
133		req			= self.connectToUrl(self.url)
134		data		= req.read()
135		checksum	= computeChecksum(data)
136		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
137
138		if checksum != self.checksum:
139			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
140
141		if not os.path.exists(os.path.dirname(dstPath)):
142			os.mkdir(os.path.dirname(dstPath))
143
144		writeBinaryFile(dstPath, data)
145
146	def extract (self):
147		print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
148
149		srcPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
150		tmpPath	= os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
151		dstPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
152
153		if self.filename.endswith(".zip"):
154			archive	= zipfile.ZipFile(srcPath)
155		else:
156			archive	= tarfile.open(srcPath)
157
158		if os.path.exists(tmpPath):
159			shutil.rmtree(tmpPath, ignore_errors=False)
160
161		os.mkdir(tmpPath)
162
163		archive.extractall(tmpPath)
164		archive.close()
165
166		extractedEntries = os.listdir(tmpPath)
167		if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
168			raise Exception("%s doesn't contain single top-level directory" % self.filename)
169
170		topLevelPath = os.path.join(tmpPath, extractedEntries[0])
171
172		if not os.path.exists(dstPath):
173			os.mkdir(dstPath)
174
175		for entry in os.listdir(topLevelPath):
176			if os.path.exists(os.path.join(dstPath, entry)):
177				raise Exception("%s exists already" % entry)
178
179			shutil.move(os.path.join(topLevelPath, entry), dstPath)
180
181		shutil.rmtree(tmpPath, ignore_errors=True)
182
183		if self.postExtract != None:
184			self.postExtract(dstPath)
185
186class SourceFile (Source):
187	def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
188		Source.__init__(self, baseDir, extractDir)
189		self.url			= url
190		self.filename		= filename
191		self.checksum		= checksum
192
193	def update (self, cmdProtocol = None, force = False):
194		if not self.isFileUpToDate():
195			Source.clean(self)
196			self.fetchAndVerifyFile()
197
198	def isFileUpToDate (self):
199		file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
200		if os.path.exists(file):
201			data = readFile(file)
202			return computeChecksum(data.encode('utf-8')) == self.checksum
203		else:
204			return False
205
206	def connectToUrl (self, url):
207		result = None
208
209		if sys.version_info < (3, 0):
210			from urllib2 import urlopen
211		else:
212			from urllib.request import urlopen
213
214		if args.insecure:
215			print("Ignoring certificate checks")
216			ssl_context = ssl._create_unverified_context()
217			result = urlopen(url, context=ssl_context)
218		else:
219			result = urlopen(url)
220
221		return result
222
223	def fetchAndVerifyFile (self):
224		print("Fetching %s" % self.url)
225
226		req			= self.connectToUrl(self.url)
227		data		= req.read()
228		checksum	= computeChecksum(data)
229		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
230
231		if checksum != self.checksum:
232			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
233
234		if not os.path.exists(os.path.dirname(dstPath)):
235			os.mkdir(os.path.dirname(dstPath))
236
237		writeBinaryFile(dstPath, data)
238
239class GitRepo (Source):
240	def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
241		Source.__init__(self, baseDir, extractDir)
242		self.httpsUrl	= httpsUrl
243		self.sshUrl		= sshUrl
244		self.revision	= revision
245		self.removeTags	= removeTags
246		self.patch		= patch
247
248	def checkout(self, url, fullDstPath, force):
249		if not os.path.exists(os.path.join(fullDstPath, '.git')):
250			execute(["git", "clone", "--no-checkout", url, fullDstPath])
251
252		pushWorkingDir(fullDstPath)
253		try:
254			for tag in self.removeTags:
255				proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
256				(stdout, stderr) = proc.communicate()
257				if len(stdout) > 0:
258					execute(["git", "tag", "-d",tag])
259			force_arg = ['--force'] if force else []
260			execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
261			execute(["git", "checkout"] + force_arg + [self.revision])
262
263			if(self.patch != ""):
264				patchFile = os.path.join(EXTERNAL_DIR, self.patch)
265				execute(["git", "reset", "--hard", "HEAD"])
266				execute(["git", "apply", patchFile])
267		finally:
268			popWorkingDir()
269
270	def update (self, cmdProtocol, force = False):
271		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
272		url         = self.httpsUrl
273		backupUrl   = self.sshUrl
274
275		# If url is none then start with ssh
276		if cmdProtocol == 'ssh' or url == None:
277			url       = self.sshUrl
278			backupUrl = self.httpsUrl
279
280		try:
281			self.checkout(url, fullDstPath, force)
282		except:
283			if backupUrl != None:
284				self.checkout(backupUrl, fullDstPath, force)
285
286def postExtractLibpng (path):
287	shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
288				os.path.join(path, "pnglibconf.h"))
289
290PACKAGES = [
291	#SourcePackage(
292	#	"http://zlib.net/fossils/zlib-1.2.13.tar.gz",
293	#	"b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
294	#	"zlib"),
295	#SourcePackage(
296	#	"http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
297	#	"c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
298	#	"libpng",
299	#	postExtract = postExtractLibpng),
300	SourcePackage(
301        {"Windows":"https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2022-05-31-12-34/ffmpeg-n4.4.2-1-g8e98dfc57f-win64-lgpl-shared-4.4.zip", "Linux": "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2022-05-31-12-34/ffmpeg-n4.4.2-1-g8e98dfc57f-linux64-gpl-shared-4.4.tar.xz", "Darwin":""}[platform.system()],
302        {"Windows":"670df8e9d2ddd5e761459b3538f64b8826566270ef1ed13bcbfc63e73aab3fd9","Linux":"817f8c93ff1ef7ede3dad15b20415d5e366bcd6848844d55046111fd3de827d0", "Darwin":""}[platform.system()],
303		"ffmpeg"),
304	#SourceFile(
305	#	"https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
306	#	"renderdoc_app.h",
307	#	"e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
308	#	"renderdoc"),
309	#GitRepo(
310	#	"https://github.com/KhronosGroup/SPIRV-Tools.git",
311	#	"git@github.com:KhronosGroup/SPIRV-Tools.git",
312	#	"f98473ceeb1d33700d01e20910433583e5256030",
313	#	"spirv-tools"),
314	#GitRepo(
315	#	"https://github.com/KhronosGroup/glslang.git",
316	#	"git@github.com:KhronosGroup/glslang.git",
317	#	"a0ad0d7067521fff880e36acfb8ce453421c3f25",
318	#	"glslang",
319	#	removeTags = ["master-tot"]),
320	#GitRepo(
321	#	"https://github.com/KhronosGroup/SPIRV-Headers.git",
322	#	"git@github.com:KhronosGroup/SPIRV-Headers.git",
323	#	"87d5b782bec60822aa878941e6b13c0a9a954c9b",
324	#	"spirv-headers"),
325	#GitRepo(
326	#	"https://github.com/KhronosGroup/Vulkan-Docs.git",
327	#	"git@github.com:KhronosGroup/Vulkan-Docs.git",
328	#	"9a2e576a052a1e65a5d41b593e693ff02745604b",
329	#	"vulkan-docs"),
330	#GitRepo(
331	#	"https://github.com/google/amber.git",
332	#	"git@github.com:google/amber.git",
333	#	"8b145a6c89dcdb4ec28173339dd176fb7b6f43ed",
334	#	"amber"),
335	#GitRepo(
336	#	"https://github.com/open-source-parsers/jsoncpp.git",
337	#	"git@github.com:open-source-parsers/jsoncpp.git",
338	#	"9059f5cad030ba11d37818847443a53918c327b1",
339	#	"jsoncpp"),
340	#GitRepo(
341	#	"https://github.com/nvpro-samples/vk_video_samples.git",
342	#	None,
343	#	"7d68747d3524842afaf050c5e00a10f5b8c07904",
344	#	"video-parser"),
345]
346
347def parseArgs ():
348	versionsForInsecure = ((2,7,9), (3,4,3))
349	versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
350
351	parser = argparse.ArgumentParser(description = "Fetch external sources")
352	parser.add_argument('--clean', dest='clean', action='store_true', default=False,
353						help='Remove sources instead of fetching')
354	parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
355						help="Disable certificate check for external sources."
356						" Minimum python version required " + versionsForInsecureStr)
357	parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
358						help="Select protocol to checkout git repositories.")
359	parser.add_argument('--force', dest='force', action='store_true', default=False,
360						help="Pass --force to git fetch and checkout commands")
361
362	args = parser.parse_args()
363
364	if args.insecure:
365		for versionItem in versionsForInsecure:
366			if (sys.version_info.major == versionItem[0]):
367				if sys.version_info < versionItem:
368					parser.error("For --insecure minimum required python version is " +
369								versionsForInsecureStr)
370				break;
371
372	return args
373
374def run(*popenargs, **kwargs):
375	process = subprocess.Popen(*popenargs, **kwargs)
376
377	try:
378		stdout, stderr = process.communicate(None)
379	except:
380		process.kill()
381		process.wait()
382		raise
383
384	retcode = process.poll()
385
386	if retcode:
387		raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
388
389	return retcode, stdout, stderr
390
391if __name__ == "__main__":
392	# Rerun script with python3 as python2 does not have lzma (xz) decompression support
393	if sys.version_info < (3, 0):
394		cmd = {"Windows": ['py', '-3'], "Linux": ['python3'], "Darwin": ['python3']}[platform.system()]
395		cmd = cmd + sys.argv
396		run(cmd)
397	else:
398		args = parseArgs()
399
400		for pkg in PACKAGES:
401			if args.clean:
402				pkg.clean()
403			else:
404				pkg.update(args.protocol, args.force)
405