1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2015 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23import os 24import re 25import sys 26import copy 27import zlib 28import time 29import shlex 30import shutil 31import fnmatch 32import tarfile 33import argparse 34import platform 35import datetime 36import tempfile 37import posixpath 38import subprocess 39 40from build.common import * 41from build.config import * 42from build.build import * 43 44pythonExecutable = sys.executable or "python" 45 46def die (msg): 47 print msg 48 sys.exit(-1) 49 50def removeLeadingPath (path, basePath): 51 # Both inputs must be normalized already 52 assert os.path.normpath(path) == path 53 assert os.path.normpath(basePath) == basePath 54 return path[len(basePath) + 1:] 55 56def findFile (candidates): 57 for file in candidates: 58 if os.path.exists(file): 59 return file 60 return None 61 62def getFileList (basePath): 63 allFiles = [] 64 basePath = os.path.normpath(basePath) 65 for root, dirs, files in os.walk(basePath): 66 for file in files: 67 relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath) 68 allFiles.append(relPath) 69 return allFiles 70 71def toDatetime (dateTuple): 72 Y, M, D = dateTuple 73 return datetime.datetime(Y, M, D) 74 75class PackageBuildInfo: 76 def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath): 77 self.releaseConfig = releaseConfig 78 self.srcBasePath = srcBasePath 79 self.dstBasePath = dstBasePath 80 self.tmpBasePath = tmpBasePath 81 82 def getReleaseConfig (self): 83 return self.releaseConfig 84 85 def getReleaseVersion (self): 86 return self.releaseConfig.getVersion() 87 88 def getReleaseId (self): 89 # Release id is crc32(releaseConfig + release) 90 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff 91 92 def getSrcBasePath (self): 93 return self.srcBasePath 94 95 def getTmpBasePath (self): 96 return self.tmpBasePath 97 98class DstFile (object): 99 def __init__ (self, dstFile): 100 self.dstFile = dstFile 101 102 def makeDir (self): 103 dirName = os.path.dirname(self.dstFile) 104 if not os.path.exists(dirName): 105 os.makedirs(dirName) 106 107 def make (self, packageBuildInfo): 108 assert False # Should not be called 109 110class CopyFile (DstFile): 111 def __init__ (self, srcFile, dstFile): 112 super(CopyFile, self).__init__(dstFile) 113 self.srcFile = srcFile 114 115 def make (self, packageBuildInfo): 116 self.makeDir() 117 if os.path.exists(self.dstFile): 118 die("%s already exists" % self.dstFile) 119 shutil.copyfile(self.srcFile, self.dstFile) 120 121class GenReleaseInfoFileTarget (DstFile): 122 def __init__ (self, dstFile): 123 super(GenReleaseInfoFileTarget, self).__init__(dstFile) 124 125 def make (self, packageBuildInfo): 126 self.makeDir() 127 128 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py")) 129 execute([ 130 pythonExecutable, 131 "-B", # no .py[co] 132 scriptPath, 133 "--name=%s" % packageBuildInfo.getReleaseVersion(), 134 "--id=0x%08x" % packageBuildInfo.getReleaseId(), 135 "--out=%s" % self.dstFile 136 ]) 137 138class GenCMake (DstFile): 139 def __init__ (self, srcFile, dstFile, replaceVars): 140 super(GenCMake, self).__init__(dstFile) 141 self.srcFile = srcFile 142 self.replaceVars = replaceVars 143 144 def make (self, packageBuildInfo): 145 self.makeDir() 146 print " GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath) 147 src = readFile(self.srcFile) 148 for var, value in self.replaceVars: 149 src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var), 150 'set(%s "%s"' % (var, value), src) 151 writeFile(self.dstFile, src) 152 153def createFileTargets (srcBasePath, dstBasePath, files, filters): 154 usedFiles = set() # Files that are already included by other filters 155 targets = [] 156 157 for isMatch, createFileObj in filters: 158 # Build list of files that match filter 159 matchingFiles = [] 160 for file in files: 161 if not file in usedFiles and isMatch(file): 162 matchingFiles.append(file) 163 164 # Build file objects, add to used set 165 for file in matchingFiles: 166 usedFiles.add(file) 167 targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file))) 168 169 return targets 170 171# Generates multiple file targets based on filters 172class FileTargetGroup: 173 def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath): 174 self.srcBasePath = srcBasePath 175 self.dstBasePath = dstBasePath 176 self.filters = filters 177 self.getSrcBasePath = srcBasePathFunc 178 179 def make (self, packageBuildInfo): 180 fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath)) 181 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath)) 182 183 allFiles = getFileList(fullSrcPath) 184 targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters) 185 186 # Make all file targets 187 for file in targets: 188 file.make(packageBuildInfo) 189 190# Single file target 191class SingleFileTarget: 192 def __init__ (self, srcFile, dstFile, makeTarget): 193 self.srcFile = srcFile 194 self.dstFile = dstFile 195 self.makeTarget = makeTarget 196 197 def make (self, packageBuildInfo): 198 fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile)) 199 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) 200 201 target = self.makeTarget(fullSrcPath, fullDstPath) 202 target.make(packageBuildInfo) 203 204class BuildTarget: 205 def __init__ (self, baseConfig, generator, targets = None): 206 self.baseConfig = baseConfig 207 self.generator = generator 208 self.targets = targets 209 210 def make (self, packageBuildInfo): 211 print " Building %s" % self.baseConfig.getBuildDir() 212 213 # Create config with full build dir path 214 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()), 215 self.baseConfig.getBuildType(), 216 self.baseConfig.getArgs(), 217 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src")) 218 219 assert not os.path.exists(config.getBuildDir()) 220 build(config, self.generator, self.targets) 221 222class BuildAndroidTarget: 223 def __init__ (self, dstFile): 224 self.dstFile = dstFile 225 226 def make (self, packageBuildInfo): 227 print " Building Android binary" 228 229 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build") 230 231 assert not os.path.exists(buildRoot) 232 os.makedirs(buildRoot) 233 234 # Execute build script 235 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py")) 236 execute([ 237 pythonExecutable, 238 "-B", # no .py[co] 239 scriptPath, 240 "--build-root=%s" % buildRoot, 241 ]) 242 243 srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk")) 244 dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) 245 246 CopyFile(srcFile, dstFile).make(packageBuildInfo) 247 248class FetchExternalSourcesTarget: 249 def __init__ (self): 250 pass 251 252 def make (self, packageBuildInfo): 253 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py")) 254 execute([ 255 pythonExecutable, 256 "-B", # no .py[co] 257 scriptPath, 258 ]) 259 260class RemoveSourcesTarget: 261 def __init__ (self): 262 pass 263 264 def make (self, packageBuildInfo): 265 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False) 266 267class Module: 268 def __init__ (self, name, targets): 269 self.name = name 270 self.targets = targets 271 272 def make (self, packageBuildInfo): 273 for target in self.targets: 274 target.make(packageBuildInfo) 275 276class ReleaseConfig: 277 def __init__ (self, name, version, modules, sources = True): 278 self.name = name 279 self.version = version 280 self.modules = modules 281 self.sources = sources 282 283 def getName (self): 284 return self.name 285 286 def getVersion (self): 287 return self.version 288 289 def getModules (self): 290 return self.modules 291 292 def packageWithSources (self): 293 return self.sources 294 295def matchIncludeExclude (includePatterns, excludePatterns, filename): 296 components = os.path.normpath(filename).split(os.sep) 297 for pattern in excludePatterns: 298 for component in components: 299 if fnmatch.fnmatch(component, pattern): 300 return False 301 302 for pattern in includePatterns: 303 for component in components: 304 if fnmatch.fnmatch(component, pattern): 305 return True 306 307 return False 308 309def copyFileFilter (includePatterns, excludePatterns=[]): 310 return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f), 311 lambda s, d: CopyFile(s, d)) 312 313def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): 314 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)]) 315 316def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): 317 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath) 318 319def makeFileCopy (srcFile, dstFile): 320 return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d)) 321 322def getReleaseFileName (configName, releaseName): 323 today = datetime.date.today() 324 return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName) 325 326def getTempDir (): 327 dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases") 328 if not os.path.exists(dirName): 329 os.makedirs(dirName) 330 return dirName 331 332def makeRelease (releaseConfig): 333 releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion()) 334 tmpPath = getTempDir() 335 srcBasePath = DEQP_DIR 336 dstBasePath = os.path.join(tmpPath, releaseName) 337 tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp") 338 packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath) 339 dstArchiveName = releaseName + ".tar.bz2" 340 341 print "Creating release %s to %s" % (releaseName, tmpPath) 342 343 # Remove old temporary dirs 344 for path in [dstBasePath, tmpBasePath]: 345 if os.path.exists(path): 346 shutil.rmtree(path, ignore_errors=False) 347 348 # Make all modules 349 for module in releaseConfig.getModules(): 350 print " Processing module %s" % module.name 351 module.make(packageBuildInfo) 352 353 # Remove sources? 354 if not releaseConfig.packageWithSources(): 355 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False) 356 357 # Create archive 358 print "Creating %s" % dstArchiveName 359 archive = tarfile.open(dstArchiveName, 'w:bz2') 360 archive.add(dstBasePath, arcname=releaseName) 361 archive.close() 362 363 # Remove tmp dirs 364 for path in [dstBasePath, tmpBasePath]: 365 if os.path.exists(path): 366 shutil.rmtree(path, ignore_errors=False) 367 368 print "Done!" 369 370# Module declarations 371 372SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"] 373TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"] 374 375BASE = Module("Base", [ 376 makeFileCopy ("LICENSE", "src/LICENSE"), 377 makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"), 378 makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS), 379 makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS), 380 makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS), 381 makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"), 382 makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]), 383 384 # Stylesheet for displaying test logs on browser 385 makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]), 386 387 # Non-optional parts of framework 388 makeFileCopy ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"), 389 makeFileCopyGroup ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS), 390 makeFileCopyGroup ("framework/common", "src/framework/common", SRC_FILE_PATTERNS), 391 makeFileCopyGroup ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS), 392 makeFileCopyGroup ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS), 393 makeFileCopyGroup ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]), 394 makeFileCopyGroup ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS), 395 396 # android sources 397 makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS), 398 makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"), 399 makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]), 400 makeFileCopyGroup ("android/scripts", "src/android/scripts", [ 401 "common.py", 402 "build.py", 403 "resources.py", 404 "install.py", 405 "launch.py", 406 "debug.py" 407 ]), 408 409 # Release info 410 GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl") 411]) 412 413DOCUMENTATION = Module("Documentation", [ 414 makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]), 415 makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]), 416]) 417 418GLSHARED = Module("Shared GL Tests", [ 419 # Optional framework components 420 makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS), 421 makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS), 422 makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS), 423 424 makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS), 425]) 426 427GLES2 = Module("GLES2", [ 428 makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS), 429 makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]), 430 makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"]) 431]) 432 433GLES3 = Module("GLES3", [ 434 makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS), 435 makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]), 436 makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"]) 437]) 438 439GLES31 = Module("GLES31", [ 440 makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS), 441 makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]), 442 makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"]) 443]) 444 445EGL = Module("EGL", [ 446 makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS) 447]) 448 449INTERNAL = Module("Internal", [ 450 makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS), 451 makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]), 452]) 453 454EXTERNAL_SRCS = Module("External sources", [ 455 FetchExternalSourcesTarget() 456]) 457 458ANDROID_BINARIES = Module("Android Binaries", [ 459 BuildAndroidTarget ("bin/android/dEQP.apk"), 460 makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]), 461]) 462 463COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))] 464NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) 465NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) 466GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) 467GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) 468 469EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"] 470 471LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [ 472 BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR), 473 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), 474 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), 475]) 476 477LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [ 478 BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR), 479 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), 480 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), 481]) 482 483# Special module to remove src dir, for example after binary build 484REMOVE_SOURCES = Module("Remove sources from package", [ 485 RemoveSourcesTarget() 486]) 487 488# Release configuration 489 490ALL_MODULES = [ 491 BASE, 492 DOCUMENTATION, 493 GLSHARED, 494 GLES2, 495 GLES3, 496 GLES31, 497 EGL, 498 INTERNAL, 499 EXTERNAL_SRCS, 500] 501 502ALL_BINARIES = [ 503 LINUX_X64_COMMON_BINARIES, 504 ANDROID_BINARIES, 505] 506 507RELEASE_CONFIGS = { 508 "src": ALL_MODULES, 509 "src-bin": ALL_MODULES + ALL_BINARIES, 510 "bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES], 511} 512 513def parseArgs (): 514 parser = argparse.ArgumentParser(description = "Build release package") 515 parser.add_argument("-c", 516 "--config", 517 dest="config", 518 choices=RELEASE_CONFIGS.keys(), 519 required=True, 520 help="Release configuration") 521 parser.add_argument("-n", 522 "--name", 523 dest="name", 524 required=True, 525 help="Package-specific name") 526 parser.add_argument("-v", 527 "--version", 528 dest="version", 529 required=True, 530 help="Version code") 531 return parser.parse_args() 532 533if __name__ == "__main__": 534 args = parseArgs() 535 config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config]) 536 makeRelease(config) 537