1#!/usr/bin/python3 2##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===## 3# 4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5# See https://llvm.org/LICENSE.txt for license information. 6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7# 8##===----------------------------------------------------------------------===## 9# 10# This script builds many different flavors of the LLVM ecosystem. It 11# will build LLVM, Clang and dragonegg as well as run tests on them. 12# This script is convenient to use to check builds and tests before 13# committing changes to the upstream repository 14# 15# A typical source setup uses three trees and looks like this: 16# 17# official 18# dragonegg 19# llvm 20# tools 21# clang 22# staging 23# dragonegg 24# llvm 25# tools 26# clang 27# commit 28# dragonegg 29# llvm 30# tools 31# clang 32# 33# In a typical workflow, the "official" tree always contains unchanged 34# sources from the main LLVM project repositories. The "staging" tree 35# is where local work is done. A set of changes resides there waiting 36# to be moved upstream. The "commit" tree is where changes from 37# "staging" make their way upstream. Individual incremental changes 38# from "staging" are applied to "commit" and committed upstream after 39# a successful build and test run. A successful build is one in which 40# testing results in no more failures than seen in the testing of the 41# "official" tree. 42# 43# A build may be invoked as such: 44# 45# llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official 46# --build=debug --build=release --build=paranoid 47# --prefix=/home/greened/install --builddir=/home/greened/build 48# 49# This will build the LLVM ecosystem, including LLVM, Clangand 50# dragonegg, putting build results in ~/build and installing tools in 51# ~/install. llvm-compilers-check creates separate build and install 52# directories for each source/build flavor. In the above example, 53# llvmbuild will build debug, release and paranoid (debug+checks) 54# flavors from each source tree (official, staging and commit) for a 55# total of nine builds. All builds will be run in parallel. 56# 57# The user may control parallelism via the --jobs and --threads 58# switches. --jobs tells llvm-compilers-check the maximum total 59# number of builds to activate in parallel. The user may think of it 60# as equivalent to the GNU make -j switch. --threads tells 61# llvm-compilers-check how many worker threads to use to accomplish 62# those builds. If --threads is less than --jobs, --threads workers 63# will be launched and each one will pick a source/flavor combination 64# to build. Then llvm-compilers-check will invoke GNU make with -j 65# (--jobs / --threads) to use up the remaining job capacity. Once a 66# worker is finished with a build, it will pick another combination 67# off the list and start building it. 68# 69##===----------------------------------------------------------------------===## 70 71import optparse 72import os 73import sys 74import threading 75import queue 76import logging 77import traceback 78import subprocess 79import re 80 81# TODO: Use shutil.which when it is available (3.2 or later) 82def find_executable(executable, path=None): 83 """Try to find 'executable' in the directories listed in 'path' (a 84 string listing directories separated by 'os.pathsep'; defaults to 85 os.environ['PATH']). Returns the complete filename or None if not 86 found 87 """ 88 if path is None: 89 path = os.environ['PATH'] 90 paths = path.split(os.pathsep) 91 extlist = [''] 92 if os.name == 'os2': 93 (base, ext) = os.path.splitext(executable) 94 # executable files on OS/2 can have an arbitrary extension, but 95 # .exe is automatically appended if no dot is present in the name 96 if not ext: 97 executable = executable + ".exe" 98 elif sys.platform == 'win32': 99 pathext = os.environ['PATHEXT'].lower().split(os.pathsep) 100 (base, ext) = os.path.splitext(executable) 101 if ext.lower() not in pathext: 102 extlist = pathext 103 for ext in extlist: 104 execname = executable + ext 105 if os.path.isfile(execname): 106 return execname 107 else: 108 for p in paths: 109 f = os.path.join(p, execname) 110 if os.path.isfile(f): 111 return f 112 else: 113 return None 114 115def is_executable(fpath): 116 return os.path.exists(fpath) and os.access(fpath, os.X_OK) 117 118def add_options(parser): 119 parser.add_option("-v", "--verbose", action="store_true", 120 default=False, 121 help=("Output informational messages" 122 " [default: %default]")) 123 parser.add_option("--src", action="append", 124 help=("Top-level source directory [default: %default]")) 125 parser.add_option("--build", action="append", 126 help=("Build types to run [default: %default]")) 127 parser.add_option("--cc", default=find_executable("cc"), 128 help=("The C compiler to use [default: %default]")) 129 parser.add_option("--cxx", default=find_executable("c++"), 130 help=("The C++ compiler to use [default: %default]")) 131 parser.add_option("--threads", default=4, type="int", 132 help=("The number of worker threads to use " 133 "[default: %default]")) 134 parser.add_option("--jobs", "-j", default=8, type="int", 135 help=("The number of simultaneous build jobs " 136 "[default: %default]")) 137 parser.add_option("--prefix", 138 help=("Root install directory [default: %default]")) 139 parser.add_option("--builddir", 140 help=("Root build directory [default: %default]")) 141 parser.add_option("--extra-llvm-config-flags", default="", 142 help=("Extra flags to pass to llvm configure [default: %default]")) 143 parser.add_option("--force-configure", default=False, action="store_true", 144 help=("Force reconfigure of all components")) 145 parser.add_option("--no-dragonegg", default=False, action="store_true", 146 help=("Do not build dragonegg")) 147 parser.add_option("--no-install", default=False, action="store_true", 148 help=("Do not do installs")) 149 parser.add_option("--keep-going", default=False, action="store_true", 150 help=("Keep going after failures")) 151 parser.add_option("--no-flavor-prefix", default=False, action="store_true", 152 help=("Do not append the build flavor to the install path")) 153 parser.add_option("--enable-werror", default=False, action="store_true", 154 help=("Build with -Werror")) 155 return 156 157def check_options(parser, options, valid_builds): 158 # See if we're building valid flavors. 159 for build in options.build: 160 if (build not in valid_builds): 161 parser.error("'" + build + "' is not a valid build flavor " 162 + str(valid_builds)) 163 164 # See if we can find source directories. 165 for src in options.src: 166 for component in components: 167 component = component.rstrip("2") 168 compsrc = src + "/" + component 169 if (not os.path.isdir(compsrc)): 170 parser.error("'" + compsrc + "' does not exist") 171 172 # See if we can find the compilers 173 options.cc = find_executable(options.cc) 174 options.cxx = find_executable(options.cxx) 175 176 return 177 178# Find a unique short name for the given set of paths. This searches 179# back through path components until it finds unique component names 180# among all given paths. 181def get_path_abbrevs(paths): 182 # Find the number of common starting characters in the last component 183 # of the paths. 184 unique_paths = list(paths) 185 186 class NotFoundException(Exception): pass 187 188 # Find a unique component of each path. 189 unique_bases = unique_paths[:] 190 found = 0 191 while len(unique_paths) > 0: 192 bases = [os.path.basename(src) for src in unique_paths] 193 components = { c for c in bases } 194 # Account for single entry in paths. 195 if len(components) > 1 or len(components) == len(bases): 196 # We found something unique. 197 for c in components: 198 if bases.count(c) == 1: 199 index = bases.index(c) 200 unique_bases[index] = c 201 # Remove the corresponding path from the set under 202 # consideration. 203 unique_paths[index] = None 204 unique_paths = [ p for p in unique_paths if p is not None ] 205 unique_paths = [os.path.dirname(src) for src in unique_paths] 206 207 if len(unique_paths) > 0: 208 raise NotFoundException() 209 210 abbrevs = dict(zip(paths, [base for base in unique_bases])) 211 212 return abbrevs 213 214# Given a set of unique names, find a short character sequence that 215# uniquely identifies them. 216def get_short_abbrevs(unique_bases): 217 # Find a unique start character for each path base. 218 my_unique_bases = unique_bases[:] 219 unique_char_starts = unique_bases[:] 220 while len(my_unique_bases) > 0: 221 for start, char_tuple in enumerate(zip(*[base 222 for base in my_unique_bases])): 223 chars = { c for c in char_tuple } 224 # Account for single path. 225 if len(chars) > 1 or len(chars) == len(char_tuple): 226 # We found something unique. 227 for c in chars: 228 if char_tuple.count(c) == 1: 229 index = char_tuple.index(c) 230 unique_char_starts[index] = start 231 # Remove the corresponding path from the set under 232 # consideration. 233 my_unique_bases[index] = None 234 my_unique_bases = [ b for b in my_unique_bases 235 if b is not None ] 236 break 237 238 if len(my_unique_bases) > 0: 239 raise NotFoundException() 240 241 abbrevs = [abbrev[start_index:start_index+3] 242 for abbrev, start_index 243 in zip([base for base in unique_bases], 244 [index for index in unique_char_starts])] 245 246 abbrevs = dict(zip(unique_bases, abbrevs)) 247 248 return abbrevs 249 250class Builder(threading.Thread): 251 class ExecutableNotFound(Exception): pass 252 class FileNotExecutable(Exception): pass 253 254 def __init__(self, work_queue, jobs, 255 build_abbrev, source_abbrev, 256 options): 257 super().__init__() 258 self.work_queue = work_queue 259 self.jobs = jobs 260 self.cc = options.cc 261 self.cxx = options.cxx 262 self.build_abbrev = build_abbrev 263 self.source_abbrev = source_abbrev 264 self.build_prefix = options.builddir 265 self.install_prefix = options.prefix 266 self.options = options 267 self.component_abbrev = dict( 268 llvm="llvm", 269 dragonegg="degg") 270 def run(self): 271 while True: 272 try: 273 source, build = self.work_queue.get() 274 self.dobuild(source, build) 275 except: 276 traceback.print_exc() 277 finally: 278 self.work_queue.task_done() 279 280 def execute(self, command, execdir, env, component): 281 prefix = self.component_abbrev[component.replace("-", "_")] 282 pwd = os.getcwd() 283 if not os.path.exists(execdir): 284 os.makedirs(execdir) 285 286 execenv = os.environ.copy() 287 288 for key, value in env.items(): 289 execenv[key] = value 290 291 self.logger.debug("[" + prefix + "] " + "env " + str(env) + " " 292 + " ".join(command)); 293 294 try: 295 proc = subprocess.Popen(command, 296 cwd=execdir, 297 env=execenv, 298 stdout=subprocess.PIPE, 299 stderr=subprocess.STDOUT) 300 301 line = proc.stdout.readline() 302 while line: 303 self.logger.info("[" + prefix + "] " 304 + str(line, "utf-8").rstrip()) 305 line = proc.stdout.readline() 306 307 (stdoutdata, stderrdata) = proc.communicate() 308 retcode = proc.wait() 309 310 return retcode 311 312 except: 313 traceback.print_exc() 314 315 # Get a list of C++ include directories to pass to clang. 316 def get_includes(self): 317 # Assume we're building with g++ for now. 318 command = [self.cxx] 319 command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"] 320 includes = [] 321 self.logger.debug(command) 322 try: 323 proc = subprocess.Popen(command, 324 stdout=subprocess.PIPE, 325 stderr=subprocess.STDOUT) 326 327 gather = False 328 line = proc.stdout.readline() 329 while line: 330 self.logger.debug(line) 331 if re.search("End of search list", str(line)) is not None: 332 self.logger.debug("Stop Gather") 333 gather = False 334 if gather: 335 includes.append(str(line, "utf-8").strip()) 336 if re.search("#include <...> search starts", str(line)) is not None: 337 self.logger.debug("Start Gather") 338 gather = True 339 line = proc.stdout.readline() 340 341 except: 342 traceback.print_exc() 343 self.logger.debug(includes) 344 return includes 345 346 def dobuild(self, source, build): 347 build_suffix = "" 348 349 ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()]) 350 351 prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]" 352 if (not self.options.no_flavor_prefix): 353 self.install_prefix += "/" + self.source_abbrev[source] + "/" + build 354 355 build_suffix += "/" + self.source_abbrev[source] + "/" + build 356 357 self.logger = logging.getLogger(prefix) 358 359 self.logger.debug(self.install_prefix) 360 361 # Assume we're building with gcc for now. 362 cxxincludes = self.get_includes() 363 cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version 364 cxxroot = os.path.dirname(cxxroot) # Remove the c++ 365 cxxroot = os.path.dirname(cxxroot) # Remove the include 366 367 configure_flags = dict( 368 llvm=dict(debug=["--prefix=" + self.install_prefix, 369 "--enable-assertions", 370 "--disable-optimized", 371 "--with-gcc-toolchain=" + cxxroot], 372 release=["--prefix=" + self.install_prefix, 373 "--enable-optimized", 374 "--with-gcc-toolchain=" + cxxroot], 375 paranoid=["--prefix=" + self.install_prefix, 376 "--enable-assertions", 377 "--enable-expensive-checks", 378 "--disable-optimized", 379 "--with-gcc-toolchain=" + cxxroot]), 380 dragonegg=dict(debug=[], 381 release=[], 382 paranoid=[])) 383 384 if (self.options.enable_werror): 385 configure_flags["llvm"]["debug"].append("--enable-werror") 386 configure_flags["llvm"]["release"].append("--enable-werror") 387 configure_flags["llvm"]["paranoid"].append("--enable-werror") 388 389 configure_env = dict( 390 llvm=dict(debug=dict(CC=self.cc, 391 CXX=self.cxx), 392 release=dict(CC=self.cc, 393 CXX=self.cxx), 394 paranoid=dict(CC=self.cc, 395 CXX=self.cxx)), 396 dragonegg=dict(debug=dict(CC=self.cc, 397 CXX=self.cxx), 398 release=dict(CC=self.cc, 399 CXX=self.cxx), 400 paranoid=dict(CC=self.cc, 401 CXX=self.cxx))) 402 403 make_flags = dict( 404 llvm=dict(debug=["-j" + str(self.jobs)], 405 release=["-j" + str(self.jobs)], 406 paranoid=["-j" + str(self.jobs)]), 407 dragonegg=dict(debug=["-j" + str(self.jobs)], 408 release=["-j" + str(self.jobs)], 409 paranoid=["-j" + str(self.jobs)])) 410 411 make_env = dict( 412 llvm=dict(debug=dict(), 413 release=dict(), 414 paranoid=dict()), 415 dragonegg=dict(debug=dict(GCC=self.cc, 416 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), 417 release=dict(GCC=self.cc, 418 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), 419 paranoid=dict(GCC=self.cc, 420 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"))) 421 422 make_install_flags = dict( 423 llvm=dict(debug=["install"], 424 release=["install"], 425 paranoid=["install"]), 426 dragonegg=dict(debug=["install"], 427 release=["install"], 428 paranoid=["install"])) 429 430 make_install_env = dict( 431 llvm=dict(debug=dict(), 432 release=dict(), 433 paranoid=dict()), 434 dragonegg=dict(debug=dict(), 435 release=dict(), 436 paranoid=dict())) 437 438 make_check_flags = dict( 439 llvm=dict(debug=["check"], 440 release=["check"], 441 paranoid=["check"]), 442 dragonegg=dict(debug=["check"], 443 release=["check"], 444 paranoid=["check"])) 445 446 make_check_env = dict( 447 llvm=dict(debug=dict(), 448 release=dict(), 449 paranoid=dict()), 450 dragonegg=dict(debug=dict(), 451 release=dict(), 452 paranoid=dict())) 453 454 for component in components: 455 comp = component[:] 456 457 if (self.options.no_dragonegg): 458 if (comp == 'dragonegg'): 459 self.logger.info("Skipping " + component + " in " 460 + builddir) 461 continue 462 463 srcdir = source + "/" + comp.rstrip("2") 464 builddir = self.build_prefix + "/" + comp + "/" + build_suffix 465 installdir = self.install_prefix 466 467 comp_key = comp.replace("-", "_") 468 469 config_args = configure_flags[comp_key][build][:] 470 config_args.extend(getattr(self.options, 471 "extra_" + comp_key.rstrip("2") 472 + "_config_flags", 473 "").split()) 474 475 self.logger.info("Configuring " + component + " in " + builddir) 476 configrc = self.configure(component, srcdir, builddir, 477 config_args, 478 configure_env[comp_key][build]) 479 480 if (configrc == None) : 481 self.logger.info("[None] Failed to configure " + component + " in " + installdir) 482 483 if (configrc == 0 or self.options.keep_going) : 484 self.logger.info("Building " + component + " in " + builddir) 485 self.logger.info("Build: make " + str(make_flags[comp_key][build])) 486 buildrc = self.make(component, srcdir, builddir, 487 make_flags[comp_key][build], 488 make_env[comp_key][build]) 489 490 if (buildrc == None) : 491 self.logger.info("[None] Failed to build " + component + " in " + installdir) 492 493 if (buildrc == 0 or self.options.keep_going) : 494 self.logger.info("Testing " + component + " in " + builddir) 495 self.logger.info("Test: make " 496 + str(make_check_flags[comp_key][build])) 497 testrc = self.make(component, srcdir, builddir, 498 make_check_flags[comp_key][build], 499 make_check_env[comp_key][build]) 500 501 if (testrc == None) : 502 self.logger.info("[None] Failed to test " + component + " in " + installdir) 503 504 if ((testrc == 0 or self.options.keep_going) 505 and not self.options.no_install): 506 self.logger.info("Installing " + component + " in " + installdir) 507 self.make(component, srcdir, builddir, 508 make_install_flags[comp_key][build], 509 make_install_env[comp_key][build]) 510 else : 511 self.logger.info("Failed testing " + component + " in " + installdir) 512 513 else : 514 self.logger.info("Failed to build " + component + " in " + installdir) 515 516 else : 517 self.logger.info("Failed to configure " + component + " in " + installdir) 518 519 def configure(self, component, srcdir, builddir, flags, env): 520 prefix = self.component_abbrev[component.replace("-", "_")] 521 522 self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> " 523 + str(builddir)) 524 525 configure_files = dict( 526 llvm=[(srcdir + "/configure", builddir + "/Makefile")], 527 dragonegg=[(None,None)]) 528 529 530 doconfig = False 531 for conf, mf in configure_files[component.replace("-", "_")]: 532 if conf is None: 533 # No configure necessary 534 return 0 535 536 if not os.path.exists(conf): 537 self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf) 538 return -1 539 540 if os.path.exists(conf) and os.path.exists(mf): 541 confstat = os.stat(conf) 542 makestat = os.stat(mf) 543 if confstat.st_mtime > makestat.st_mtime: 544 doconfig = True 545 break 546 else: 547 doconfig = True 548 break 549 550 if not doconfig and not self.options.force_configure: 551 return 0 552 553 program = srcdir + "/configure" 554 if not is_executable(program): 555 self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program) 556 return -1 557 558 args = [program] 559 args += ["--verbose"] 560 args += flags 561 return self.execute(args, builddir, env, component) 562 563 def make(self, component, srcdir, builddir, flags, env): 564 program = find_executable("make") 565 if program is None: 566 raise ExecutableNotFound 567 568 if not is_executable(program): 569 raise FileNotExecutable 570 571 args = [program] 572 args += flags 573 return self.execute(args, builddir, env, component) 574 575# Global constants 576build_abbrev = dict(debug="dbg", release="opt", paranoid="par") 577components = ["llvm", "dragonegg"] 578 579# Parse options 580parser = optparse.OptionParser(version="%prog 1.0") 581add_options(parser) 582(options, args) = parser.parse_args() 583check_options(parser, options, build_abbrev.keys()); 584 585if options.verbose: 586 logging.basicConfig(level=logging.DEBUG, 587 format='%(name)-13s: %(message)s') 588else: 589 logging.basicConfig(level=logging.INFO, 590 format='%(name)-13s: %(message)s') 591 592source_abbrev = get_path_abbrevs(set(options.src)) 593 594work_queue = queue.Queue() 595 596jobs = options.jobs // options.threads 597if jobs == 0: 598 jobs = 1 599 600numthreads = options.threads 601 602logging.getLogger().info("Building with " + str(options.jobs) + " jobs and " 603 + str(numthreads) + " threads using " + str(jobs) 604 + " make jobs") 605 606logging.getLogger().info("CC = " + str(options.cc)) 607logging.getLogger().info("CXX = " + str(options.cxx)) 608 609for t in range(numthreads): 610 builder = Builder(work_queue, jobs, 611 build_abbrev, source_abbrev, 612 options) 613 builder.daemon = True 614 builder.start() 615 616for build in set(options.build): 617 for source in set(options.src): 618 work_queue.put((source, build)) 619 620work_queue.join() 621