1#!/usr/bin/python3 2##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===## 3# 4# The LLVM Compiler Infrastructure 5# 6# This file is distributed under the University of Illinois Open Source 7# License. See LICENSE.TXT for details. 8# 9##===----------------------------------------------------------------------===## 10# 11# This script builds many different flavors of the LLVM ecosystem. It 12# will build LLVM, Clang and dragonegg as well as run tests on them. 13# This script is convenient to use to check builds and tests before 14# committing changes to the upstream repository 15# 16# A typical source setup uses three trees and looks like this: 17# 18# official 19# dragonegg 20# llvm 21# tools 22# clang 23# staging 24# dragonegg 25# llvm 26# tools 27# clang 28# commit 29# dragonegg 30# llvm 31# tools 32# clang 33# 34# In a typical workflow, the "official" tree always contains unchanged 35# sources from the main LLVM project repositories. The "staging" tree 36# is where local work is done. A set of changes resides there waiting 37# to be moved upstream. The "commit" tree is where changes from 38# "staging" make their way upstream. Individual incremental changes 39# from "staging" are applied to "commit" and committed upstream after 40# a successful build and test run. A successful build is one in which 41# testing results in no more failures than seen in the testing of the 42# "official" tree. 43# 44# A build may be invoked as such: 45# 46# llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official 47# --build=debug --build=release --build=paranoid 48# --prefix=/home/greened/install --builddir=/home/greened/build 49# 50# This will build the LLVM ecosystem, including LLVM, Clangand 51# dragonegg, putting build results in ~/build and installing tools in 52# ~/install. llvm-compilers-check creates separate build and install 53# directories for each source/build flavor. In the above example, 54# llvmbuild will build debug, release and paranoid (debug+checks) 55# flavors from each source tree (official, staging and commit) for a 56# total of nine builds. All builds will be run in parallel. 57# 58# The user may control parallelism via the --jobs and --threads 59# switches. --jobs tells llvm-compilers-checl the maximum total 60# number of builds to activate in parallel. The user may think of it 61# as equivalent to the GNU make -j switch. --threads tells 62# llvm-compilers-check how many worker threads to use to accomplish 63# those builds. If --threads is less than --jobs, --threads workers 64# will be launched and each one will pick a source/flavor combination 65# to build. Then llvm-compilers-check will invoke GNU make with -j 66# (--jobs / --threads) to use up the remaining job capacity. Once a 67# worker is finished with a build, it will pick another combination 68# off the list and start building it. 69# 70##===----------------------------------------------------------------------===## 71 72import optparse 73import os 74import sys 75import threading 76import queue 77import logging 78import traceback 79import subprocess 80import re 81 82# TODO: Use shutil.which when it is available (3.2 or later) 83def find_executable(executable, path=None): 84 """Try to find 'executable' in the directories listed in 'path' (a 85 string listing directories separated by 'os.pathsep'; defaults to 86 os.environ['PATH']). Returns the complete filename or None if not 87 found 88 """ 89 if path is None: 90 path = os.environ['PATH'] 91 paths = path.split(os.pathsep) 92 extlist = [''] 93 if os.name == 'os2': 94 (base, ext) = os.path.splitext(executable) 95 # executable files on OS/2 can have an arbitrary extension, but 96 # .exe is automatically appended if no dot is present in the name 97 if not ext: 98 executable = executable + ".exe" 99 elif sys.platform == 'win32': 100 pathext = os.environ['PATHEXT'].lower().split(os.pathsep) 101 (base, ext) = os.path.splitext(executable) 102 if ext.lower() not in pathext: 103 extlist = pathext 104 for ext in extlist: 105 execname = executable + ext 106 if os.path.isfile(execname): 107 return execname 108 else: 109 for p in paths: 110 f = os.path.join(p, execname) 111 if os.path.isfile(f): 112 return f 113 else: 114 return None 115 116def is_executable(fpath): 117 return os.path.exists(fpath) and os.access(fpath, os.X_OK) 118 119def add_options(parser): 120 parser.add_option("-v", "--verbose", action="store_true", 121 default=False, 122 help=("Output informational messages" 123 " [default: %default]")) 124 parser.add_option("--src", action="append", 125 help=("Top-level source directory [default: %default]")) 126 parser.add_option("--build", action="append", 127 help=("Build types to run [default: %default]")) 128 parser.add_option("--cc", default=find_executable("cc"), 129 help=("The C compiler to use [default: %default]")) 130 parser.add_option("--cxx", default=find_executable("c++"), 131 help=("The C++ compiler to use [default: %default]")) 132 parser.add_option("--threads", default=4, type="int", 133 help=("The number of worker threads to use " 134 "[default: %default]")) 135 parser.add_option("--jobs", "-j", default=8, type="int", 136 help=("The number of simultaneous build jobs " 137 "[default: %default]")) 138 parser.add_option("--prefix", 139 help=("Root install directory [default: %default]")) 140 parser.add_option("--builddir", 141 help=("Root build directory [default: %default]")) 142 parser.add_option("--extra-llvm-config-flags", default="", 143 help=("Extra flags to pass to llvm configure [default: %default]")) 144 parser.add_option("--force-configure", default=False, action="store_true", 145 help=("Force reconfigure of all components")) 146 parser.add_option("--no-dragonegg", default=False, action="store_true", 147 help=("Do not build dragonegg")) 148 parser.add_option("--no-install", default=False, action="store_true", 149 help=("Do not do installs")) 150 return 151 152def check_options(parser, options, valid_builds): 153 # See if we're building valid flavors. 154 for build in options.build: 155 if (build not in valid_builds): 156 parser.error("'" + build + "' is not a valid build flavor " 157 + str(valid_builds)) 158 159 # See if we can find source directories. 160 for src in options.src: 161 for component in components: 162 component = component.rstrip("2") 163 compsrc = src + "/" + component 164 if (not os.path.isdir(compsrc)): 165 parser.error("'" + compsrc + "' does not exist") 166 167 # See if we can find the compilers 168 options.cc = find_executable(options.cc) 169 options.cxx = find_executable(options.cxx) 170 171 return 172 173# Find a unique short name for the given set of paths. This searches 174# back through path components until it finds unique component names 175# among all given paths. 176def get_path_abbrevs(paths): 177 # Find the number of common starting characters in the last component 178 # of the paths. 179 unique_paths = list(paths) 180 181 class NotFoundException(Exception): pass 182 183 # Find a unique component of each path. 184 unique_bases = unique_paths[:] 185 found = 0 186 while len(unique_paths) > 0: 187 bases = [os.path.basename(src) for src in unique_paths] 188 components = { c for c in bases } 189 # Account for single entry in paths. 190 if len(components) > 1 or len(components) == len(bases): 191 # We found something unique. 192 for c in components: 193 if bases.count(c) == 1: 194 index = bases.index(c) 195 unique_bases[index] = c 196 # Remove the corresponding path from the set under 197 # consideration. 198 unique_paths[index] = None 199 unique_paths = [ p for p in unique_paths if p is not None ] 200 unique_paths = [os.path.dirname(src) for src in unique_paths] 201 202 if len(unique_paths) > 0: 203 raise NotFoundException() 204 205 abbrevs = dict(zip(paths, [base for base in unique_bases])) 206 207 return abbrevs 208 209# Given a set of unique names, find a short character sequence that 210# uniquely identifies them. 211def get_short_abbrevs(unique_bases): 212 # Find a unique start character for each path base. 213 my_unique_bases = unique_bases[:] 214 unique_char_starts = unique_bases[:] 215 while len(my_unique_bases) > 0: 216 for start, char_tuple in enumerate(zip(*[base 217 for base in my_unique_bases])): 218 chars = { c for c in char_tuple } 219 # Account for single path. 220 if len(chars) > 1 or len(chars) == len(char_tuple): 221 # We found something unique. 222 for c in chars: 223 if char_tuple.count(c) == 1: 224 index = char_tuple.index(c) 225 unique_char_starts[index] = start 226 # Remove the corresponding path from the set under 227 # consideration. 228 my_unique_bases[index] = None 229 my_unique_bases = [ b for b in my_unique_bases 230 if b is not None ] 231 break 232 233 if len(my_unique_bases) > 0: 234 raise NotFoundException() 235 236 abbrevs = [abbrev[start_index:start_index+3] 237 for abbrev, start_index 238 in zip([base for base in unique_bases], 239 [index for index in unique_char_starts])] 240 241 abbrevs = dict(zip(unique_bases, abbrevs)) 242 243 return abbrevs 244 245class Builder(threading.Thread): 246 class ExecutableNotFound(Exception): pass 247 class FileNotExecutable(Exception): pass 248 249 def __init__(self, work_queue, jobs, 250 build_abbrev, source_abbrev, 251 options): 252 super().__init__() 253 self.work_queue = work_queue 254 self.jobs = jobs 255 self.cc = options.cc 256 self.cxx = options.cxx 257 self.build_abbrev = build_abbrev 258 self.source_abbrev = source_abbrev 259 self.build_prefix = options.builddir 260 self.install_prefix = options.prefix 261 self.options = options 262 self.component_abbrev = dict( 263 llvm="llvm", 264 dragonegg="degg") 265 def run(self): 266 while True: 267 try: 268 source, build = self.work_queue.get() 269 self.dobuild(source, build) 270 except: 271 traceback.print_exc() 272 finally: 273 self.work_queue.task_done() 274 275 def execute(self, command, execdir, env, component): 276 prefix = self.component_abbrev[component.replace("-", "_")] 277 pwd = os.getcwd() 278 if not os.path.exists(execdir): 279 os.makedirs(execdir) 280 281 execenv = os.environ.copy() 282 283 for key, value in env.items(): 284 execenv[key] = value 285 286 self.logger.debug("[" + prefix + "] " + "env " + str(env) + " " 287 + " ".join(command)); 288 289 try: 290 proc = subprocess.Popen(command, 291 cwd=execdir, 292 env=execenv, 293 stdout=subprocess.PIPE, 294 stderr=subprocess.STDOUT) 295 296 line = proc.stdout.readline() 297 while line: 298 self.logger.info("[" + prefix + "] " 299 + str(line, "utf-8").rstrip()) 300 line = proc.stdout.readline() 301 302 except: 303 traceback.print_exc() 304 305 # Get a list of C++ include directories to pass to clang. 306 def get_includes(self): 307 # Assume we're building with g++ for now. 308 command = [self.cxx] 309 command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"] 310 includes = [] 311 self.logger.debug(command) 312 try: 313 proc = subprocess.Popen(command, 314 stdout=subprocess.PIPE, 315 stderr=subprocess.STDOUT) 316 317 gather = False 318 line = proc.stdout.readline() 319 while line: 320 self.logger.debug(line) 321 if re.search("End of search list", str(line)) is not None: 322 self.logger.debug("Stop Gather") 323 gather = False 324 if gather: 325 includes.append(str(line, "utf-8").strip()) 326 if re.search("#include <...> search starts", str(line)) is not None: 327 self.logger.debug("Start Gather") 328 gather = True 329 line = proc.stdout.readline() 330 except: 331 traceback.print_exc() 332 self.logger.debug(includes) 333 return includes 334 335 def dobuild(self, source, build): 336 build_suffix = "" 337 338 ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()]) 339 340 prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]" 341 self.install_prefix += "/" + self.source_abbrev[source] + "/" + build 342 build_suffix += "/" + self.source_abbrev[source] + "/" + build 343 344 self.logger = logging.getLogger(prefix) 345 346 self.logger.debug(self.install_prefix) 347 348 # Assume we're building with gcc for now. 349 cxxincludes = self.get_includes() 350 cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version 351 cxxroot = os.path.dirname(cxxroot) # Remove the c++ 352 cxxroot = os.path.dirname(cxxroot) # Remove the include 353 354 configure_flags = dict( 355 llvm=dict(debug=["--prefix=" + self.install_prefix, 356 "--with-extra-options=-Werror", 357 "--enable-assertions", 358 "--disable-optimized", 359 "--with-gcc-toolchain=" + cxxroot], 360 release=["--prefix=" + self.install_prefix, 361 "--with-extra-options=-Werror", 362 "--enable-optimized", 363 "--with-gcc-toolchain=" + cxxroot], 364 paranoid=["--prefix=" + self.install_prefix, 365 "--with-extra-options=-Werror", 366 "--enable-assertions", 367 "--enable-expensive-checks", 368 "--disable-optimized", 369 "--with-gcc-toolchain=" + cxxroot]), 370 dragonegg=dict(debug=[], 371 release=[], 372 paranoid=[])) 373 374 configure_env = dict( 375 llvm=dict(debug=dict(CC=self.cc, 376 CXX=self.cxx), 377 release=dict(CC=self.cc, 378 CXX=self.cxx), 379 paranoid=dict(CC=self.cc, 380 CXX=self.cxx)), 381 dragonegg=dict(debug=dict(CC=self.cc, 382 CXX=self.cxx), 383 release=dict(CC=self.cc, 384 CXX=self.cxx), 385 paranoid=dict(CC=self.cc, 386 CXX=self.cxx))) 387 388 make_flags = dict( 389 llvm=dict(debug=["-j" + str(self.jobs)], 390 release=["-j" + str(self.jobs)], 391 paranoid=["-j" + str(self.jobs)]), 392 dragonegg=dict(debug=["-j" + str(self.jobs)], 393 release=["-j" + str(self.jobs)], 394 paranoid=["-j" + str(self.jobs)])) 395 396 make_env = dict( 397 llvm=dict(debug=dict(), 398 release=dict(), 399 paranoid=dict()), 400 dragonegg=dict(debug=dict(GCC=self.cc, 401 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), 402 release=dict(GCC=self.cc, 403 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), 404 paranoid=dict(GCC=self.cc, 405 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"))) 406 407 make_install_flags = dict( 408 llvm=dict(debug=["install"], 409 release=["install"], 410 paranoid=["install"]), 411 dragonegg=dict(debug=["install"], 412 release=["install"], 413 paranoid=["install"])) 414 415 make_install_env = dict( 416 llvm=dict(debug=dict(), 417 release=dict(), 418 paranoid=dict()), 419 dragonegg=dict(debug=dict(), 420 release=dict(), 421 paranoid=dict())) 422 423 make_check_flags = dict( 424 llvm=dict(debug=["check"], 425 release=["check"], 426 paranoid=["check"]), 427 dragonegg=dict(debug=["check"], 428 release=["check"], 429 paranoid=["check"])) 430 431 make_check_env = dict( 432 llvm=dict(debug=dict(), 433 release=dict(), 434 paranoid=dict()), 435 dragonegg=dict(debug=dict(), 436 release=dict(), 437 paranoid=dict())) 438 439 for component in components: 440 comp = component[:] 441 442 if (self.options.no_dragonegg): 443 if (comp == 'dragonegg'): 444 self.logger.info("Skipping " + component + " in " 445 + builddir) 446 continue 447 448 srcdir = source + "/" + comp.rstrip("2") 449 builddir = self.build_prefix + "/" + comp + "/" + build_suffix 450 installdir = self.install_prefix 451 452 comp_key = comp.replace("-", "_") 453 454 config_args = configure_flags[comp_key][build][:] 455 config_args.extend(getattr(self.options, 456 "extra_" + comp_key.rstrip("2") 457 + "_config_flags", 458 "").split()) 459 460 self.logger.info("Configuring " + component + " in " + builddir) 461 self.configure(component, srcdir, builddir, 462 config_args, 463 configure_env[comp_key][build]) 464 465 self.logger.info("Building " + component + " in " + builddir) 466 self.logger.info("Build: make " + str(make_flags[comp_key][build])) 467 self.make(component, srcdir, builddir, 468 make_flags[comp_key][build], 469 make_env[comp_key][build]) 470 471 if (not self.options.no_install): 472 self.logger.info("Installing " + component + " in " + installdir) 473 self.make(component, srcdir, builddir, 474 make_install_flags[comp_key][build], 475 make_install_env[comp_key][build]) 476 477 self.logger.info("Testing " + component + " in " + builddir) 478 self.logger.info("Test: make " 479 + str(make_check_flags[comp_key][build])) 480 self.make(component, srcdir, builddir, 481 make_check_flags[comp_key][build], 482 make_check_env[comp_key][build]) 483 484 485 def configure(self, component, srcdir, builddir, flags, env): 486 self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> " 487 + str(builddir)) 488 489 configure_files = dict( 490 llvm=[(srcdir + "/configure", builddir + "/Makefile")], 491 dragonegg=[("","")]) 492 493 494 doconfig = False 495 for conf, mf in configure_files[component.replace("-", "_")]: 496 if not os.path.exists(conf): 497 return 498 if os.path.exists(conf) and os.path.exists(mf): 499 confstat = os.stat(conf) 500 makestat = os.stat(mf) 501 if confstat.st_mtime > makestat.st_mtime: 502 doconfig = True 503 break 504 else: 505 doconfig = True 506 break 507 508 if not doconfig and not self.options.force_configure: 509 return 510 511 program = srcdir + "/configure" 512 if not is_executable(program): 513 return 514 515 args = [program] 516 args += ["--verbose"] 517 args += flags 518 self.execute(args, builddir, env, component) 519 520 def make(self, component, srcdir, builddir, flags, env): 521 program = find_executable("make") 522 if program is None: 523 raise ExecutableNotFound 524 525 if not is_executable(program): 526 raise FileNotExecutable 527 528 args = [program] 529 args += flags 530 self.execute(args, builddir, env, component) 531 532# Global constants 533build_abbrev = dict(debug="dbg", release="opt", paranoid="par") 534components = ["llvm", "dragonegg"] 535 536# Parse options 537parser = optparse.OptionParser(version="%prog 1.0") 538add_options(parser) 539(options, args) = parser.parse_args() 540check_options(parser, options, build_abbrev.keys()); 541 542if options.verbose: 543 logging.basicConfig(level=logging.DEBUG, 544 format='%(name)-13s: %(message)s') 545else: 546 logging.basicConfig(level=logging.INFO, 547 format='%(name)-13s: %(message)s') 548 549source_abbrev = get_path_abbrevs(set(options.src)) 550 551work_queue = queue.Queue() 552 553jobs = options.jobs // options.threads 554if jobs == 0: 555 jobs = 1 556 557numthreads = options.threads 558 559logging.getLogger().info("Building with " + str(options.jobs) + " jobs and " 560 + str(numthreads) + " threads using " + str(jobs) 561 + " make jobs") 562 563logging.getLogger().info("CC = " + str(options.cc)) 564logging.getLogger().info("CXX = " + str(options.cxx)) 565 566for t in range(numthreads): 567 builder = Builder(work_queue, jobs, 568 build_abbrev, source_abbrev, 569 options) 570 builder.daemon = True 571 builder.start() 572 573for build in set(options.build): 574 for source in set(options.src): 575 work_queue.put((source, build)) 576 577work_queue.join() 578