1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2015 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23import os 24import re 25import sys 26import copy 27import zlib 28import time 29import shlex 30import shutil 31import fnmatch 32import tarfile 33import argparse 34import platform 35import datetime 36import tempfile 37import posixpath 38import subprocess 39 40from build.common import * 41from build.config import * 42from build.build import * 43 44def die (msg): 45 print msg 46 sys.exit(-1) 47 48def removeLeadingPath (path, basePath): 49 # Both inputs must be normalized already 50 assert os.path.normpath(path) == path 51 assert os.path.normpath(basePath) == basePath 52 return path[len(basePath) + 1:] 53 54def findFile (candidates): 55 for file in candidates: 56 if os.path.exists(file): 57 return file 58 return None 59 60def getFileList (basePath): 61 allFiles = [] 62 basePath = os.path.normpath(basePath) 63 for root, dirs, files in os.walk(basePath): 64 for file in files: 65 relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath) 66 allFiles.append(relPath) 67 return allFiles 68 69def toDatetime (dateTuple): 70 Y, M, D = dateTuple 71 return datetime.datetime(Y, M, D) 72 73class PackageBuildInfo: 74 def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath): 75 self.releaseConfig = releaseConfig 76 self.srcBasePath = srcBasePath 77 self.dstBasePath = dstBasePath 78 self.tmpBasePath = tmpBasePath 79 80 def getReleaseConfig (self): 81 return self.releaseConfig 82 83 def getReleaseVersion (self): 84 return self.releaseConfig.getVersion() 85 86 def getReleaseId (self): 87 # Release id is crc32(releaseConfig + release) 88 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff 89 90 def getSrcBasePath (self): 91 return self.srcBasePath 92 93 def getTmpBasePath (self): 94 return self.tmpBasePath 95 96class DstFile (object): 97 def __init__ (self, dstFile): 98 self.dstFile = dstFile 99 100 def makeDir (self): 101 dirName = os.path.dirname(self.dstFile) 102 if not os.path.exists(dirName): 103 os.makedirs(dirName) 104 105 def make (self, packageBuildInfo): 106 assert False # Should not be called 107 108class CopyFile (DstFile): 109 def __init__ (self, srcFile, dstFile): 110 super(CopyFile, self).__init__(dstFile) 111 self.srcFile = srcFile 112 113 def make (self, packageBuildInfo): 114 self.makeDir() 115 if os.path.exists(self.dstFile): 116 die("%s already exists" % self.dstFile) 117 shutil.copyfile(self.srcFile, self.dstFile) 118 119class GenReleaseInfoFileTarget (DstFile): 120 def __init__ (self, dstFile): 121 super(GenReleaseInfoFileTarget, self).__init__(dstFile) 122 123 def make (self, packageBuildInfo): 124 self.makeDir() 125 126 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py")) 127 execute([ 128 "python", 129 "-B", # no .py[co] 130 scriptPath, 131 "--name=%s" % packageBuildInfo.getReleaseVersion(), 132 "--id=0x%08x" % packageBuildInfo.getReleaseId(), 133 "--out=%s" % self.dstFile 134 ]) 135 136class GenCMake (DstFile): 137 def __init__ (self, srcFile, dstFile, replaceVars): 138 super(GenCMake, self).__init__(dstFile) 139 self.srcFile = srcFile 140 self.replaceVars = replaceVars 141 142 def make (self, packageBuildInfo): 143 self.makeDir() 144 print " GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath) 145 src = readFile(self.srcFile) 146 for var, value in self.replaceVars: 147 src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var), 148 'set(%s "%s"' % (var, value), src) 149 writeFile(self.dstFile, src) 150 151def createFileTargets (srcBasePath, dstBasePath, files, filters): 152 usedFiles = set() # Files that are already included by other filters 153 targets = [] 154 155 for isMatch, createFileObj in filters: 156 # Build list of files that match filter 157 matchingFiles = [] 158 for file in files: 159 if not file in usedFiles and isMatch(file): 160 matchingFiles.append(file) 161 162 # Build file objects, add to used set 163 for file in matchingFiles: 164 usedFiles.add(file) 165 targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file))) 166 167 return targets 168 169# Generates multiple file targets based on filters 170class FileTargetGroup: 171 def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath): 172 self.srcBasePath = srcBasePath 173 self.dstBasePath = dstBasePath 174 self.filters = filters 175 self.getSrcBasePath = srcBasePathFunc 176 177 def make (self, packageBuildInfo): 178 fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath)) 179 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath)) 180 181 allFiles = getFileList(fullSrcPath) 182 targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters) 183 184 # Make all file targets 185 for file in targets: 186 file.make(packageBuildInfo) 187 188# Single file target 189class SingleFileTarget: 190 def __init__ (self, srcFile, dstFile, makeTarget): 191 self.srcFile = srcFile 192 self.dstFile = dstFile 193 self.makeTarget = makeTarget 194 195 def make (self, packageBuildInfo): 196 fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile)) 197 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) 198 199 target = self.makeTarget(fullSrcPath, fullDstPath) 200 target.make(packageBuildInfo) 201 202class BuildTarget: 203 def __init__ (self, baseConfig, generator, targets = None): 204 self.baseConfig = baseConfig 205 self.generator = generator 206 self.targets = targets 207 208 def make (self, packageBuildInfo): 209 print " Building %s" % self.baseConfig.getBuildDir() 210 211 # Create config with full build dir path 212 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()), 213 self.baseConfig.getBuildType(), 214 self.baseConfig.getArgs(), 215 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src")) 216 217 assert not os.path.exists(config.getBuildDir()) 218 build(config, self.generator, self.targets) 219 220class BuildAndroidTarget: 221 def __init__ (self, dstFile): 222 self.dstFile = dstFile 223 224 def make (self, packageBuildInfo): 225 print " Building Android binary" 226 227 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build") 228 229 assert not os.path.exists(buildRoot) 230 os.makedirs(buildRoot) 231 232 # Execute build script 233 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py")) 234 execute([ 235 "python", 236 "-B", # no .py[co] 237 scriptPath, 238 "--build-root=%s" % buildRoot, 239 ]) 240 241 srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk")) 242 dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) 243 244 CopyFile(srcFile, dstFile).make(packageBuildInfo) 245 246class FetchExternalSourcesTarget: 247 def __init__ (self): 248 pass 249 250 def make (self, packageBuildInfo): 251 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py")) 252 execute([ 253 "python", 254 "-B", # no .py[co] 255 scriptPath, 256 ]) 257 258class RemoveSourcesTarget: 259 def __init__ (self): 260 pass 261 262 def make (self, packageBuildInfo): 263 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False) 264 265class Module: 266 def __init__ (self, name, targets): 267 self.name = name 268 self.targets = targets 269 270 def make (self, packageBuildInfo): 271 for target in self.targets: 272 target.make(packageBuildInfo) 273 274class ReleaseConfig: 275 def __init__ (self, name, version, modules, sources = True): 276 self.name = name 277 self.version = version 278 self.modules = modules 279 self.sources = sources 280 281 def getName (self): 282 return self.name 283 284 def getVersion (self): 285 return self.version 286 287 def getModules (self): 288 return self.modules 289 290 def packageWithSources (self): 291 return self.sources 292 293def matchIncludeExclude (includePatterns, excludePatterns, filename): 294 components = os.path.normpath(filename).split(os.sep) 295 for pattern in excludePatterns: 296 for component in components: 297 if fnmatch.fnmatch(component, pattern): 298 return False 299 300 for pattern in includePatterns: 301 for component in components: 302 if fnmatch.fnmatch(component, pattern): 303 return True 304 305 return False 306 307def copyFileFilter (includePatterns, excludePatterns=[]): 308 return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f), 309 lambda s, d: CopyFile(s, d)) 310 311def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): 312 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)]) 313 314def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): 315 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath) 316 317def makeFileCopy (srcFile, dstFile): 318 return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d)) 319 320def getReleaseFileName (configName, releaseName): 321 today = datetime.date.today() 322 return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName) 323 324def getTempDir (): 325 dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases") 326 if not os.path.exists(dirName): 327 os.makedirs(dirName) 328 return dirName 329 330def makeRelease (releaseConfig): 331 releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion()) 332 tmpPath = getTempDir() 333 srcBasePath = DEQP_DIR 334 dstBasePath = os.path.join(tmpPath, releaseName) 335 tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp") 336 packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath) 337 dstArchiveName = releaseName + ".tar.bz2" 338 339 print "Creating release %s to %s" % (releaseName, tmpPath) 340 341 # Remove old temporary dirs 342 for path in [dstBasePath, tmpBasePath]: 343 if os.path.exists(path): 344 shutil.rmtree(path, ignore_errors=False) 345 346 # Make all modules 347 for module in releaseConfig.getModules(): 348 print " Processing module %s" % module.name 349 module.make(packageBuildInfo) 350 351 # Remove sources? 352 if not releaseConfig.packageWithSources(): 353 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False) 354 355 # Create archive 356 print "Creating %s" % dstArchiveName 357 archive = tarfile.open(dstArchiveName, 'w:bz2') 358 archive.add(dstBasePath, arcname=releaseName) 359 archive.close() 360 361 # Remove tmp dirs 362 for path in [dstBasePath, tmpBasePath]: 363 if os.path.exists(path): 364 shutil.rmtree(path, ignore_errors=False) 365 366 print "Done!" 367 368# Module declarations 369 370SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"] 371TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"] 372 373BASE = Module("Base", [ 374 makeFileCopy ("LICENSE", "src/LICENSE"), 375 makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"), 376 makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS), 377 makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS), 378 makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS), 379 makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"), 380 makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]), 381 382 # Stylesheet for displaying test logs on browser 383 makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]), 384 385 # Non-optional parts of framework 386 makeFileCopy ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"), 387 makeFileCopyGroup ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS), 388 makeFileCopyGroup ("framework/common", "src/framework/common", SRC_FILE_PATTERNS), 389 makeFileCopyGroup ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS), 390 makeFileCopyGroup ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS), 391 makeFileCopyGroup ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]), 392 makeFileCopyGroup ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS), 393 394 # android sources 395 makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS), 396 makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"), 397 makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]), 398 makeFileCopyGroup ("android/scripts", "src/android/scripts", [ 399 "common.py", 400 "build.py", 401 "resources.py", 402 "install.py", 403 "launch.py", 404 "debug.py" 405 ]), 406 407 # Release info 408 GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl") 409]) 410 411DOCUMENTATION = Module("Documentation", [ 412 makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]), 413 makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]), 414]) 415 416GLSHARED = Module("Shared GL Tests", [ 417 # Optional framework components 418 makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS), 419 makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS), 420 makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS), 421 422 makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS), 423]) 424 425GLES2 = Module("GLES2", [ 426 makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS), 427 makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]), 428 makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"]) 429]) 430 431GLES3 = Module("GLES3", [ 432 makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS), 433 makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]), 434 makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"]) 435]) 436 437GLES31 = Module("GLES31", [ 438 makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS), 439 makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]), 440 makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"]) 441]) 442 443EGL = Module("EGL", [ 444 makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS) 445]) 446 447INTERNAL = Module("Internal", [ 448 makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS), 449 makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]), 450]) 451 452EXTERNAL_SRCS = Module("External sources", [ 453 FetchExternalSourcesTarget() 454]) 455 456ANDROID_BINARIES = Module("Android Binaries", [ 457 BuildAndroidTarget ("bin/android/dEQP.apk"), 458 makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]), 459]) 460 461COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))] 462NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) 463NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) 464GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) 465GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) 466 467EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"] 468 469LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [ 470 BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR), 471 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), 472 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), 473]) 474 475LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [ 476 BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR), 477 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), 478 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), 479]) 480 481# Special module to remove src dir, for example after binary build 482REMOVE_SOURCES = Module("Remove sources from package", [ 483 RemoveSourcesTarget() 484]) 485 486# Release configuration 487 488ALL_MODULES = [ 489 BASE, 490 DOCUMENTATION, 491 GLSHARED, 492 GLES2, 493 GLES3, 494 GLES31, 495 EGL, 496 INTERNAL, 497 EXTERNAL_SRCS, 498] 499 500ALL_BINARIES = [ 501 LINUX_X64_COMMON_BINARIES, 502 ANDROID_BINARIES, 503] 504 505RELEASE_CONFIGS = { 506 "src": ALL_MODULES, 507 "src-bin": ALL_MODULES + ALL_BINARIES, 508 "bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES], 509} 510 511def parseArgs (): 512 parser = argparse.ArgumentParser(description = "Build release package") 513 parser.add_argument("-c", 514 "--config", 515 dest="config", 516 choices=RELEASE_CONFIGS.keys(), 517 required=True, 518 help="Release configuration") 519 parser.add_argument("-n", 520 "--name", 521 dest="name", 522 required=True, 523 help="Package-specific name") 524 parser.add_argument("-v", 525 "--version", 526 dest="version", 527 required=True, 528 help="Version code") 529 return parser.parse_args() 530 531if __name__ == "__main__": 532 args = parseArgs() 533 config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config]) 534 makeRelease(config) 535