1#!/usr/bin/env python 2""" 3This script is used to build "official" universal installers on macOS. 4 5NEW for 3.6.5: 6- support Intel 64-bit-only () and 32-bit-only installer builds 7- build and link with private Tcl/Tk 8.6 for 10.9+ builds 8- deprecate use of explicit SDK (--sdk-path=) since all but the oldest 9 versions of Xcode support implicit setting of an SDK via environment 10 variables (SDKROOT and friends, see the xcrun man page for more info). 11 The SDK stuff was primarily needed for building universal installers 12 for 10.4; so as of 3.6.5, building installers for 10.4 is no longer 13 supported with build-installer. 14- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" 15 16Please ensure that this script keeps working with Python 2.5, to avoid 17bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Doc builds 18use current versions of Sphinx and require a reasonably current python3. 19Sphinx and dependencies are installed into a venv using the python3's pip 20so will fetch them from PyPI if necessary. Since python3 is now used for 21Sphinx, build-installer.py should also be converted to use python3! 22 23For 10.9 or greater deployment targets, build-installer builds and links 24with its own copy of Tcl/Tk 8.5 and the rest of this paragraph does not 25apply. Otherwise, build-installer requires an installed third-party version 26of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 27(for 10.6 or later) installed in /Library/Frameworks. When installed, 28the Python built by this script will attempt to dynamically link first to 29Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall 30back to the ones in /System/Library/Framework. For the build, we recommend 31installing the most recent ActiveTcl 8.5 or 8.4 version, depending 32on the deployment target. The actual version linked to depends on the 33path of /Library/Frameworks/{Tcl,Tk}.framework/Versions/Current. 34 35Usage: see USAGE variable in the script. 36""" 37import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp 38try: 39 import urllib2 as urllib_request 40except ImportError: 41 import urllib.request as urllib_request 42 43STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 44 | stat.S_IRGRP | stat.S_IXGRP 45 | stat.S_IROTH | stat.S_IXOTH ) 46 47STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 48 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP 49 | stat.S_IROTH | stat.S_IXOTH ) 50 51INCLUDE_TIMESTAMP = 1 52VERBOSE = 1 53 54from plistlib import Plist 55 56try: 57 from plistlib import writePlist 58except ImportError: 59 # We're run using python2.3 60 def writePlist(plist, path): 61 plist.write(path) 62 63def shellQuote(value): 64 """ 65 Return the string value in a form that can safely be inserted into 66 a shell command. 67 """ 68 return "'%s'"%(value.replace("'", "'\"'\"'")) 69 70def grepValue(fn, variable): 71 """ 72 Return the unquoted value of a variable from a file.. 73 QUOTED_VALUE='quotes' -> str('quotes') 74 UNQUOTED_VALUE=noquotes -> str('noquotes') 75 """ 76 variable = variable + '=' 77 for ln in open(fn, 'r'): 78 if ln.startswith(variable): 79 value = ln[len(variable):].strip() 80 return value.strip("\"'") 81 raise RuntimeError("Cannot find variable %s" % variable[:-1]) 82 83_cache_getVersion = None 84 85def getVersion(): 86 global _cache_getVersion 87 if _cache_getVersion is None: 88 _cache_getVersion = grepValue( 89 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION') 90 return _cache_getVersion 91 92def getVersionMajorMinor(): 93 return tuple([int(n) for n in getVersion().split('.', 2)]) 94 95_cache_getFullVersion = None 96 97def getFullVersion(): 98 global _cache_getFullVersion 99 if _cache_getFullVersion is not None: 100 return _cache_getFullVersion 101 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h') 102 for ln in open(fn): 103 if 'PY_VERSION' in ln: 104 _cache_getFullVersion = ln.split()[-1][1:-1] 105 return _cache_getFullVersion 106 raise RuntimeError("Cannot find full version??") 107 108FW_PREFIX = ["Library", "Frameworks", "Python.framework"] 109FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions 110FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions 111 112# The directory we'll use to create the build (will be erased and recreated) 113WORKDIR = "/tmp/_py" 114 115# The directory we'll use to store third-party sources. Set this to something 116# else if you don't want to re-fetch required libraries every time. 117DEPSRC = os.path.join(WORKDIR, 'third-party') 118DEPSRC = os.path.expanduser('~/Universal/other-sources') 119 120universal_opts_map = { '32-bit': ('i386', 'ppc',), 121 '64-bit': ('x86_64', 'ppc64',), 122 'intel': ('i386', 'x86_64'), 123 'intel-32': ('i386',), 124 'intel-64': ('x86_64',), 125 '3-way': ('ppc', 'i386', 'x86_64'), 126 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } 127default_target_map = { 128 '64-bit': '10.5', 129 '3-way': '10.5', 130 'intel': '10.5', 131 'intel-32': '10.4', 132 'intel-64': '10.5', 133 'all': '10.5', 134} 135 136UNIVERSALOPTS = tuple(universal_opts_map.keys()) 137 138UNIVERSALARCHS = '32-bit' 139 140ARCHLIST = universal_opts_map[UNIVERSALARCHS] 141 142# Source directory (assume we're in Mac/BuildScript) 143SRCDIR = os.path.dirname( 144 os.path.dirname( 145 os.path.dirname( 146 os.path.abspath(__file__ 147 )))) 148 149# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level 150DEPTARGET = '10.5' 151 152def getDeptargetTuple(): 153 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) 154 155def getTargetCompilers(): 156 target_cc_map = { 157 '10.4': ('gcc-4.0', 'g++-4.0'), 158 '10.5': ('gcc', 'g++'), 159 '10.6': ('gcc', 'g++'), 160 } 161 return target_cc_map.get(DEPTARGET, ('gcc', 'g++') ) 162 163CC, CXX = getTargetCompilers() 164 165PYTHON_2 = getVersionMajorMinor()[0] == 2 166PYTHON_3 = getVersionMajorMinor()[0] == 3 167 168USAGE = textwrap.dedent("""\ 169 Usage: build_python [options] 170 171 Options: 172 -? or -h: Show this message 173 -b DIR 174 --build-dir=DIR: Create build here (default: %(WORKDIR)r) 175 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) 176 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) 177 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) 178 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) 179 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) 180""")% globals() 181 182# Dict of object file names with shared library names to check after building. 183# This is to ensure that we ended up dynamically linking with the shared 184# library paths and versions we expected. For example: 185# EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 186# '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl', 187# '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] 188EXPECTED_SHARED_LIBS = {} 189 190# Are we building and linking with our own copy of Tcl/TK? 191# For now, do so if deployment target is 10.9+. 192def internalTk(): 193 return getDeptargetTuple() >= (10, 9) 194 195# List of names of third party software built with this installer. 196# The names will be inserted into the rtf version of the License. 197THIRD_PARTY_LIBS = [] 198 199# Instructions for building libraries that are necessary for building a 200# batteries included python. 201# [The recipes are defined here for convenience but instantiated later after 202# command line options have been processed.] 203def library_recipes(): 204 result = [] 205 206 LT_10_5 = bool(getDeptargetTuple() < (10, 5)) 207 208 # Since Apple removed the header files for the deprecated system 209 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not 210 # have much choice but to build our own copy here, too. 211 212 result.extend([ 213 dict( 214 name="OpenSSL 1.0.2p", 215 url="https://www.openssl.org/source/openssl-1.0.2p.tar.gz", 216 checksum='ac5eb30bf5798aa14b1ae6d0e7da58df', 217 buildrecipe=build_universal_openssl, 218 configure=None, 219 install=None, 220 ), 221 ]) 222 223 if internalTk(): 224 result.extend([ 225 dict( 226 name="Tcl 8.6.8", 227 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz", 228 checksum='81656d3367af032e0ae6157eff134f89', 229 buildDir="unix", 230 configure_pre=[ 231 '--enable-shared', 232 '--enable-threads', 233 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 234 ], 235 useLDFlags=False, 236 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 237 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 238 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 239 }, 240 ), 241 dict( 242 name="Tk 8.6.8", 243 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz", 244 checksum='5e0faecba458ee1386078fb228d008ba', 245 patches=[ 246 "tk868_on_10_8_10_9.patch", 247 ], 248 buildDir="unix", 249 configure_pre=[ 250 '--enable-aqua', 251 '--enable-shared', 252 '--enable-threads', 253 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 254 ], 255 useLDFlags=False, 256 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 257 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 258 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 259 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), 260 }, 261 ), 262 ]) 263 264 if PYTHON_3: 265 result.extend([ 266 dict( 267 name="XZ 5.2.3", 268 url="http://tukaani.org/xz/xz-5.2.3.tar.gz", 269 checksum='ef68674fb47a8b8e741b34e429d86e9d', 270 configure_pre=[ 271 '--disable-dependency-tracking', 272 ] 273 ), 274 ]) 275 276 result.extend([ 277 dict( 278 name="NCurses 5.9", 279 url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz", 280 checksum='8cb9c412e5f2d96bc6f459aa8c6282a1', 281 configure_pre=[ 282 "--enable-widec", 283 "--without-cxx", 284 "--without-cxx-binding", 285 "--without-ada", 286 "--without-curses-h", 287 "--enable-shared", 288 "--with-shared", 289 "--without-debug", 290 "--without-normal", 291 "--without-tests", 292 "--without-manpages", 293 "--datadir=/usr/share", 294 "--sysconfdir=/etc", 295 "--sharedstatedir=/usr/com", 296 "--with-terminfo-dirs=/usr/share/terminfo", 297 "--with-default-terminfo-dir=/usr/share/terminfo", 298 "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), 299 ], 300 patchscripts=[ 301 ("ftp://invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", 302 "f54bf02a349f96a7c4f0d00922f3a0d4"), 303 ], 304 useLDFlags=False, 305 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%( 306 shellQuote(os.path.join(WORKDIR, 'libraries')), 307 shellQuote(os.path.join(WORKDIR, 'libraries')), 308 getVersion(), 309 ), 310 ), 311 dict( 312 name="SQLite 3.22.0", 313 url="https://www.sqlite.org/2018/sqlite-autoconf-3220000.tar.gz", 314 checksum='96b5648d542e8afa6ab7ffb8db8ddc3d', 315 extra_cflags=('-Os ' 316 '-DSQLITE_ENABLE_FTS5 ' 317 '-DSQLITE_ENABLE_FTS4 ' 318 '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' 319 '-DSQLITE_ENABLE_JSON1 ' 320 '-DSQLITE_ENABLE_RTREE ' 321 '-DSQLITE_TCL=0 ' 322 '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), 323 configure_pre=[ 324 '--enable-threadsafe', 325 '--enable-shared=no', 326 '--enable-static=yes', 327 '--disable-readline', 328 '--disable-dependency-tracking', 329 ] 330 ), 331 ]) 332 333 if getDeptargetTuple() < (10, 5): 334 result.extend([ 335 dict( 336 name="Bzip2 1.0.6", 337 url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", 338 checksum='00b516f4704d4a7cb50a1d97e6e8e15b', 339 configure=None, 340 install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( 341 CC, CXX, 342 shellQuote(os.path.join(WORKDIR, 'libraries')), 343 ' -arch '.join(ARCHLIST), 344 ), 345 ), 346 dict( 347 name="ZLib 1.2.3", 348 url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", 349 checksum='debc62758716a169df9f62e6ab2bc634', 350 configure=None, 351 install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( 352 CC, CXX, 353 shellQuote(os.path.join(WORKDIR, 'libraries')), 354 ' -arch '.join(ARCHLIST), 355 ), 356 ), 357 dict( 358 # Note that GNU readline is GPL'd software 359 name="GNU Readline 6.1.2", 360 url="http://ftp.gnu.org/pub/gnu/readline/readline-6.1.tar.gz" , 361 checksum='fc2f7e714fe792db1ce6ddc4c9fb4ef3', 362 patchlevel='0', 363 patches=[ 364 # The readline maintainers don't do actual micro releases, but 365 # just ship a set of patches. 366 ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-001', 367 'c642f2e84d820884b0bf9fd176bc6c3f'), 368 ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-002', 369 '1a76781a1ea734e831588285db7ec9b1'), 370 ] 371 ), 372 ]) 373 374 if not PYTHON_3: 375 result.extend([ 376 dict( 377 name="Sleepycat DB 4.7.25", 378 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz", 379 checksum='ec2b87e833779681a0c3a814aa71359e', 380 buildDir="build_unix", 381 configure="../dist/configure", 382 configure_pre=[ 383 '--includedir=/usr/local/include/db4', 384 ] 385 ), 386 ]) 387 388 return result 389 390 391# Instructions for building packages inside the .mpkg. 392def pkg_recipes(): 393 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3] 394 result = [ 395 dict( 396 name="PythonFramework", 397 long_name="Python Framework", 398 source="/Library/Frameworks/Python.framework", 399 readme="""\ 400 This package installs Python.framework, that is the python 401 interpreter and the standard library. 402 """, 403 postflight="scripts/postflight.framework", 404 selected='selected', 405 ), 406 dict( 407 name="PythonApplications", 408 long_name="GUI Applications", 409 source="/Applications/Python %(VER)s", 410 readme="""\ 411 This package installs IDLE (an interactive Python IDE), 412 Python Launcher and Build Applet (create application bundles 413 from python scripts). 414 415 It also installs a number of examples and demos. 416 """, 417 required=False, 418 selected='selected', 419 ), 420 dict( 421 name="PythonUnixTools", 422 long_name="UNIX command-line tools", 423 source="/usr/local/bin", 424 readme="""\ 425 This package installs the unix tools in /usr/local/bin for 426 compatibility with older releases of Python. This package 427 is not necessary to use Python. 428 """, 429 required=False, 430 selected='selected', 431 ), 432 dict( 433 name="PythonDocumentation", 434 long_name="Python Documentation", 435 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation", 436 source="/pydocs", 437 readme="""\ 438 This package installs the python documentation at a location 439 that is useable for pydoc and IDLE. 440 """, 441 postflight="scripts/postflight.documentation", 442 required=False, 443 selected='selected', 444 ), 445 dict( 446 name="PythonProfileChanges", 447 long_name="Shell profile updater", 448 readme="""\ 449 This packages updates your shell profile to make sure that 450 the Python tools are found by your shell in preference of 451 the system provided Python tools. 452 453 If you don't install this package you'll have to add 454 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin" 455 to your PATH by hand. 456 """, 457 postflight="scripts/postflight.patch-profile", 458 topdir="/Library/Frameworks/Python.framework", 459 source="/empty-dir", 460 required=False, 461 selected='selected', 462 ), 463 dict( 464 name="PythonInstallPip", 465 long_name="Install or upgrade pip", 466 readme="""\ 467 This package installs (or upgrades from an earlier version) 468 pip, a tool for installing and managing Python packages. 469 """, 470 postflight="scripts/postflight.ensurepip", 471 topdir="/Library/Frameworks/Python.framework", 472 source="/empty-dir", 473 required=False, 474 selected='selected', 475 ), 476 ] 477 478 return result 479 480def fatal(msg): 481 """ 482 A fatal error, bail out. 483 """ 484 sys.stderr.write('FATAL: ') 485 sys.stderr.write(msg) 486 sys.stderr.write('\n') 487 sys.exit(1) 488 489def fileContents(fn): 490 """ 491 Return the contents of the named file 492 """ 493 return open(fn, 'r').read() 494 495def runCommand(commandline): 496 """ 497 Run a command and raise RuntimeError if it fails. Output is suppressed 498 unless the command fails. 499 """ 500 fd = os.popen(commandline, 'r') 501 data = fd.read() 502 xit = fd.close() 503 if xit is not None: 504 sys.stdout.write(data) 505 raise RuntimeError("command failed: %s"%(commandline,)) 506 507 if VERBOSE: 508 sys.stdout.write(data); sys.stdout.flush() 509 510def captureCommand(commandline): 511 fd = os.popen(commandline, 'r') 512 data = fd.read() 513 xit = fd.close() 514 if xit is not None: 515 sys.stdout.write(data) 516 raise RuntimeError("command failed: %s"%(commandline,)) 517 518 return data 519 520def getTclTkVersion(configfile, versionline): 521 """ 522 search Tcl or Tk configuration file for version line 523 """ 524 try: 525 f = open(configfile, "r") 526 except OSError: 527 fatal("Framework configuration file not found: %s" % configfile) 528 529 for l in f: 530 if l.startswith(versionline): 531 f.close() 532 return l 533 534 fatal("Version variable %s not found in framework configuration file: %s" 535 % (versionline, configfile)) 536 537def checkEnvironment(): 538 """ 539 Check that we're running on a supported system. 540 """ 541 542 if sys.version_info[0:2] < (2, 5): 543 fatal("This script must be run with Python 2.5 (or later)") 544 545 if platform.system() != 'Darwin': 546 fatal("This script should be run on a macOS 10.5 (or later) system") 547 548 if int(platform.release().split('.')[0]) < 8: 549 fatal("This script should be run on a macOS 10.5 (or later) system") 550 551 # Because we only support dynamic load of only one major/minor version of 552 # Tcl/Tk, if we are not using building and using our own private copy of 553 # Tcl/Tk, ensure: 554 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked 555 # in) SDKROOT/Library/Frameworks. As of Python 3.6.5, we no longer 556 # enforce that the version of the user-installed framework also 557 # exists in the system-supplied Tcl/Tk frameworks. Time to support 558 # Tcl/Tk 8.6 even if Apple does not. 559 if not internalTk(): 560 frameworks = {} 561 for framework in ['Tcl', 'Tk']: 562 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework 563 libfw = os.path.join('/', fwpth) 564 usrfw = os.path.join(os.getenv('HOME'), fwpth) 565 frameworks[framework] = os.readlink(libfw) 566 if not os.path.exists(libfw): 567 fatal("Please install a link to a current %s %s as %s so " 568 "the user can override the system framework." 569 % (framework, frameworks[framework], libfw)) 570 if os.path.exists(usrfw): 571 fatal("Please rename %s to avoid possible dynamic load issues." 572 % usrfw) 573 574 if frameworks['Tcl'] != frameworks['Tk']: 575 fatal("The Tcl and Tk frameworks are not the same version.") 576 577 print(" -- Building with external Tcl/Tk %s frameworks" 578 % frameworks['Tk']) 579 580 # add files to check after build 581 EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 582 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" 583 % frameworks['Tcl'], 584 "/Library/Frameworks/Tk.framework/Versions/%s/Tk" 585 % frameworks['Tk'], 586 ] 587 else: 588 print(" -- Building private copy of Tcl/Tk") 589 print("") 590 591 # Remove inherited environment variables which might influence build 592 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 593 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] 594 for ev in list(os.environ): 595 for prefix in environ_var_prefixes: 596 if ev.startswith(prefix) : 597 print("INFO: deleting environment variable %s=%s" % ( 598 ev, os.environ[ev])) 599 del os.environ[ev] 600 601 base_path = '/bin:/sbin:/usr/bin:/usr/sbin' 602 if 'SDK_TOOLS_BIN' in os.environ: 603 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path 604 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; 605 # add its fixed location here if it exists 606 OLD_DEVELOPER_TOOLS = '/Developer/Tools' 607 if os.path.isdir(OLD_DEVELOPER_TOOLS): 608 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS 609 os.environ['PATH'] = base_path 610 print("Setting default PATH: %s"%(os.environ['PATH'])) 611 if PYTHON_2: 612 # Ensure we have access to sphinx-build. 613 # You may have to define SDK_TOOLS_BIN and link to it there, 614 runCommand('sphinx-build --version') 615 616def parseOptions(args=None): 617 """ 618 Parse arguments and update global settings. 619 """ 620 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET 621 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX 622 global FW_VERSION_PREFIX 623 global FW_SSL_DIRECTORY 624 625 if args is None: 626 args = sys.argv[1:] 627 628 try: 629 options, args = getopt.getopt(args, '?hb', 630 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', 631 'dep-target=', 'universal-archs=', 'help' ]) 632 except getopt.GetoptError: 633 print(sys.exc_info()[1]) 634 sys.exit(1) 635 636 if args: 637 print("Additional arguments") 638 sys.exit(1) 639 640 deptarget = None 641 for k, v in options: 642 if k in ('-h', '-?', '--help'): 643 print(USAGE) 644 sys.exit(0) 645 646 elif k in ('-d', '--build-dir'): 647 WORKDIR=v 648 649 elif k in ('--third-party',): 650 DEPSRC=v 651 652 elif k in ('--sdk-path',): 653 print(" WARNING: --sdk-path is no longer supported") 654 655 elif k in ('--src-dir',): 656 SRCDIR=v 657 658 elif k in ('--dep-target', ): 659 DEPTARGET=v 660 deptarget=v 661 662 elif k in ('--universal-archs', ): 663 if v in UNIVERSALOPTS: 664 UNIVERSALARCHS = v 665 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 666 if deptarget is None: 667 # Select alternate default deployment 668 # target 669 DEPTARGET = default_target_map.get(v, '10.5') 670 else: 671 raise NotImplementedError(v) 672 673 else: 674 raise NotImplementedError(k) 675 676 SRCDIR=os.path.abspath(SRCDIR) 677 WORKDIR=os.path.abspath(WORKDIR) 678 DEPSRC=os.path.abspath(DEPSRC) 679 680 CC, CXX = getTargetCompilers() 681 682 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()] 683 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"] 684 685 print("-- Settings:") 686 print(" * Source directory: %s" % SRCDIR) 687 print(" * Build directory: %s" % WORKDIR) 688 print(" * Third-party source: %s" % DEPSRC) 689 print(" * Deployment target: %s" % DEPTARGET) 690 print(" * Universal archs: %s" % str(ARCHLIST)) 691 print(" * C compiler: %s" % CC) 692 print(" * C++ compiler: %s" % CXX) 693 print("") 694 print(" -- Building a Python %s framework at patch level %s" 695 % (getVersion(), getFullVersion())) 696 print("") 697 698def extractArchive(builddir, archiveName): 699 """ 700 Extract a source archive into 'builddir'. Returns the path of the 701 extracted archive. 702 703 XXX: This function assumes that archives contain a toplevel directory 704 that is has the same name as the basename of the archive. This is 705 safe enough for almost anything we use. Unfortunately, it does not 706 work for current Tcl and Tk source releases where the basename of 707 the archive ends with "-src" but the uncompressed directory does not. 708 For now, just special case Tcl and Tk tar.gz downloads. 709 """ 710 curdir = os.getcwd() 711 try: 712 os.chdir(builddir) 713 if archiveName.endswith('.tar.gz'): 714 retval = os.path.basename(archiveName[:-7]) 715 if ((retval.startswith('tcl') or retval.startswith('tk')) 716 and retval.endswith('-src')): 717 retval = retval[:-4] 718 if os.path.exists(retval): 719 shutil.rmtree(retval) 720 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') 721 722 elif archiveName.endswith('.tar.bz2'): 723 retval = os.path.basename(archiveName[:-8]) 724 if os.path.exists(retval): 725 shutil.rmtree(retval) 726 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r') 727 728 elif archiveName.endswith('.tar'): 729 retval = os.path.basename(archiveName[:-4]) 730 if os.path.exists(retval): 731 shutil.rmtree(retval) 732 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r') 733 734 elif archiveName.endswith('.zip'): 735 retval = os.path.basename(archiveName[:-4]) 736 if os.path.exists(retval): 737 shutil.rmtree(retval) 738 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r') 739 740 data = fp.read() 741 xit = fp.close() 742 if xit is not None: 743 sys.stdout.write(data) 744 raise RuntimeError("Cannot extract %s"%(archiveName,)) 745 746 return os.path.join(builddir, retval) 747 748 finally: 749 os.chdir(curdir) 750 751def downloadURL(url, fname): 752 """ 753 Download the contents of the url into the file. 754 """ 755 fpIn = urllib_request.urlopen(url) 756 fpOut = open(fname, 'wb') 757 block = fpIn.read(10240) 758 try: 759 while block: 760 fpOut.write(block) 761 block = fpIn.read(10240) 762 fpIn.close() 763 fpOut.close() 764 except: 765 try: 766 os.unlink(fname) 767 except OSError: 768 pass 769 770def verifyThirdPartyFile(url, checksum, fname): 771 """ 772 Download file from url to filename fname if it does not already exist. 773 Abort if file contents does not match supplied md5 checksum. 774 """ 775 name = os.path.basename(fname) 776 if os.path.exists(fname): 777 print("Using local copy of %s"%(name,)) 778 else: 779 print("Did not find local copy of %s"%(name,)) 780 print("Downloading %s"%(name,)) 781 downloadURL(url, fname) 782 print("Archive for %s stored as %s"%(name, fname)) 783 if os.system( 784 'MD5=$(openssl md5 %s) ; test "${MD5##*= }" = "%s"' 785 % (shellQuote(fname), checksum) ): 786 fatal('MD5 checksum mismatch for file %s' % fname) 787 788def build_universal_openssl(basedir, archList): 789 """ 790 Special case build recipe for universal build of openssl. 791 792 The upstream OpenSSL build system does not directly support 793 OS X universal builds. We need to build each architecture 794 separately then lipo them together into fat libraries. 795 """ 796 797 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4). 798 # If we are building on a 10.4.x or earlier system, 799 # unilaterally disable assembly code building to avoid the problem. 800 no_asm = int(platform.release().split(".")[0]) < 9 801 802 def build_openssl_arch(archbase, arch): 803 "Build one architecture of openssl" 804 arch_opts = { 805 "i386": ["darwin-i386-cc"], 806 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], 807 "ppc": ["darwin-ppc-cc"], 808 "ppc64": ["darwin64-ppc-cc"], 809 } 810 configure_opts = [ 811 "no-krb5", 812 "no-idea", 813 "no-mdc2", 814 "no-rc5", 815 "no-zlib", 816 "enable-tlsext", 817 "no-ssl2", 818 "no-ssl3", 819 # "enable-unit-test", 820 "shared", 821 "--install_prefix=%s"%shellQuote(archbase), 822 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), 823 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), 824 ] 825 if no_asm: 826 configure_opts.append("no-asm") 827 # OpenSSL 1.0.2o broke the Configure test for whether the compiler 828 # in use supports dependency rule generation (cc -M) with gcc-4.2 829 # used for the 10.6+ installer builds. Patch Configure here to 830 # force use of "cc -M" rather than "makedepend". 831 runCommand( 832 """sed -i "" 's|my $cc_as_makedepend = 0|my $cc_as_makedepend = 1|g' Configure""") 833 834 runCommand(" ".join(["perl", "Configure"] 835 + arch_opts[arch] + configure_opts)) 836 runCommand("make depend") 837 runCommand("make all") 838 runCommand("make install_sw") 839 # runCommand("make test") 840 return 841 842 srcdir = os.getcwd() 843 universalbase = os.path.join(srcdir, "..", 844 os.path.basename(srcdir) + "-universal") 845 os.mkdir(universalbase) 846 archbasefws = [] 847 for arch in archList: 848 # fresh copy of the source tree 849 archsrc = os.path.join(universalbase, arch, "src") 850 shutil.copytree(srcdir, archsrc, symlinks=True) 851 # install base for this arch 852 archbase = os.path.join(universalbase, arch, "root") 853 os.mkdir(archbase) 854 # Python framework base within install_prefix: 855 # the build will install into this framework.. 856 # This is to ensure that the resulting shared libs have 857 # the desired real install paths built into them. 858 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX) 859 860 # build one architecture 861 os.chdir(archsrc) 862 build_openssl_arch(archbase, arch) 863 os.chdir(srcdir) 864 archbasefws.append(archbasefw) 865 866 # copy arch-independent files from last build into the basedir framework 867 basefw = os.path.join(basedir, *FW_VERSION_PREFIX) 868 shutil.copytree( 869 os.path.join(archbasefw, "include", "openssl"), 870 os.path.join(basefw, "include", "openssl") 871 ) 872 873 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"), 874 "SHLIB_VERSION_NUMBER") 875 # e.g. -> "1.0.0" 876 libcrypto = "libcrypto.dylib" 877 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".") 878 # e.g. -> "libcrypto.1.0.0.dylib" 879 libssl = "libssl.dylib" 880 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".") 881 # e.g. -> "libssl.1.0.0.dylib" 882 883 try: 884 os.mkdir(os.path.join(basefw, "lib")) 885 except OSError: 886 pass 887 888 # merge the individual arch-dependent shared libs into a fat shared lib 889 archbasefws.insert(0, basefw) 890 for (lib_unversioned, lib_versioned) in [ 891 (libcrypto, libcrypto_versioned), 892 (libssl, libssl_versioned) 893 ]: 894 runCommand("lipo -create -output " + 895 " ".join(shellQuote( 896 os.path.join(fw, "lib", lib_versioned)) 897 for fw in archbasefws)) 898 # and create an unversioned symlink of it 899 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned)) 900 901 # Create links in the temp include and lib dirs that will be injected 902 # into the Python build so that setup.py can find them while building 903 # and the versioned links so that the setup.py post-build import test 904 # does not fail. 905 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX) 906 for fn in [ 907 ["include", "openssl"], 908 ["lib", libcrypto], 909 ["lib", libssl], 910 ["lib", libcrypto_versioned], 911 ["lib", libssl_versioned], 912 ]: 913 os.symlink( 914 os.path.join(relative_path, *fn), 915 os.path.join(basedir, "usr", "local", *fn) 916 ) 917 918 return 919 920def buildRecipe(recipe, basedir, archList): 921 """ 922 Build software using a recipe. This function does the 923 'configure;make;make install' dance for C software, with a possibility 924 to customize this process, basically a poor-mans DarwinPorts. 925 """ 926 curdir = os.getcwd() 927 928 name = recipe['name'] 929 THIRD_PARTY_LIBS.append(name) 930 url = recipe['url'] 931 configure = recipe.get('configure', './configure') 932 buildrecipe = recipe.get('buildrecipe', None) 933 install = recipe.get('install', 'make && make install DESTDIR=%s'%( 934 shellQuote(basedir))) 935 936 archiveName = os.path.split(url)[-1] 937 sourceArchive = os.path.join(DEPSRC, archiveName) 938 939 if not os.path.exists(DEPSRC): 940 os.mkdir(DEPSRC) 941 942 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive) 943 print("Extracting archive for %s"%(name,)) 944 buildDir=os.path.join(WORKDIR, '_bld') 945 if not os.path.exists(buildDir): 946 os.mkdir(buildDir) 947 948 workDir = extractArchive(buildDir, sourceArchive) 949 os.chdir(workDir) 950 951 for patch in recipe.get('patches', ()): 952 if isinstance(patch, tuple): 953 url, checksum = patch 954 fn = os.path.join(DEPSRC, os.path.basename(url)) 955 verifyThirdPartyFile(url, checksum, fn) 956 else: 957 # patch is a file in the source directory 958 fn = os.path.join(curdir, patch) 959 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1), 960 shellQuote(fn),)) 961 962 for patchscript in recipe.get('patchscripts', ()): 963 if isinstance(patchscript, tuple): 964 url, checksum = patchscript 965 fn = os.path.join(DEPSRC, os.path.basename(url)) 966 verifyThirdPartyFile(url, checksum, fn) 967 else: 968 # patch is a file in the source directory 969 fn = os.path.join(curdir, patchscript) 970 if fn.endswith('.bz2'): 971 runCommand('bunzip2 -fk %s' % shellQuote(fn)) 972 fn = fn[:-4] 973 runCommand('sh %s' % shellQuote(fn)) 974 os.unlink(fn) 975 976 if 'buildDir' in recipe: 977 os.chdir(recipe['buildDir']) 978 979 if configure is not None: 980 configure_args = [ 981 "--prefix=/usr/local", 982 "--enable-static", 983 "--disable-shared", 984 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),), 985 ] 986 987 if 'configure_pre' in recipe: 988 args = list(recipe['configure_pre']) 989 if '--disable-static' in args: 990 configure_args.remove('--enable-static') 991 if '--enable-shared' in args: 992 configure_args.remove('--disable-shared') 993 configure_args.extend(args) 994 995 if recipe.get('useLDFlags', 1): 996 configure_args.extend([ 997 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 998 "-I%s/usr/local/include"%( 999 recipe.get('extra_cflags', ''), 1000 DEPTARGET, 1001 ' -arch '.join(archList), 1002 shellQuote(basedir)[1:-1],), 1003 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( 1004 DEPTARGET, 1005 shellQuote(basedir)[1:-1], 1006 ' -arch '.join(archList)), 1007 ]) 1008 else: 1009 configure_args.extend([ 1010 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1011 "-I%s/usr/local/include"%( 1012 recipe.get('extra_cflags', ''), 1013 DEPTARGET, 1014 ' -arch '.join(archList), 1015 shellQuote(basedir)[1:-1],), 1016 ]) 1017 1018 if 'configure_post' in recipe: 1019 configure_args = configure_args + list(recipe['configure_post']) 1020 1021 configure_args.insert(0, configure) 1022 configure_args = [ shellQuote(a) for a in configure_args ] 1023 1024 print("Running configure for %s"%(name,)) 1025 runCommand(' '.join(configure_args) + ' 2>&1') 1026 1027 if buildrecipe is not None: 1028 # call special-case build recipe, e.g. for openssl 1029 buildrecipe(basedir, archList) 1030 1031 if install is not None: 1032 print("Running install for %s"%(name,)) 1033 runCommand('{ ' + install + ' ;} 2>&1') 1034 1035 print("Done %s"%(name,)) 1036 print("") 1037 1038 os.chdir(curdir) 1039 1040def buildLibraries(): 1041 """ 1042 Build our dependencies into $WORKDIR/libraries/usr/local 1043 """ 1044 print("") 1045 print("Building required libraries") 1046 print("") 1047 universal = os.path.join(WORKDIR, 'libraries') 1048 os.mkdir(universal) 1049 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib')) 1050 os.makedirs(os.path.join(universal, 'usr', 'local', 'include')) 1051 1052 for recipe in library_recipes(): 1053 buildRecipe(recipe, universal, ARCHLIST) 1054 1055 1056 1057def buildPythonDocs(): 1058 # This stores the documentation as Resources/English.lproj/Documentation 1059 # inside the framework. pydoc and IDLE will pick it up there. 1060 print("Install python documentation") 1061 rootDir = os.path.join(WORKDIR, '_root') 1062 buildDir = os.path.join('../../Doc') 1063 docdir = os.path.join(rootDir, 'pydocs') 1064 curDir = os.getcwd() 1065 os.chdir(buildDir) 1066 runCommand('make clean') 1067 if PYTHON_2: 1068 # Python 2 doc builds do not use blurb nor do they have a venv target. 1069 # Assume sphinx-build is on our PATH, checked in checkEnvironment 1070 runCommand('make html') 1071 else: 1072 # Create virtual environment for docs builds with blurb and sphinx 1073 runCommand('make venv') 1074 runCommand('make html PYTHON=venv/bin/python') 1075 os.chdir(curDir) 1076 if not os.path.exists(docdir): 1077 os.mkdir(docdir) 1078 os.rename(os.path.join(buildDir, 'build', 'html'), docdir) 1079 1080 1081def buildPython(): 1082 print("Building a universal python for %s architectures" % UNIVERSALARCHS) 1083 1084 buildDir = os.path.join(WORKDIR, '_bld', 'python') 1085 rootDir = os.path.join(WORKDIR, '_root') 1086 1087 if os.path.exists(buildDir): 1088 shutil.rmtree(buildDir) 1089 if os.path.exists(rootDir): 1090 shutil.rmtree(rootDir) 1091 os.makedirs(buildDir) 1092 os.makedirs(rootDir) 1093 os.makedirs(os.path.join(rootDir, 'empty-dir')) 1094 curdir = os.getcwd() 1095 os.chdir(buildDir) 1096 1097 # Extract the version from the configure file, needed to calculate 1098 # several paths. 1099 version = getVersion() 1100 1101 # Since the extra libs are not in their installed framework location 1102 # during the build, augment the library path so that the interpreter 1103 # will find them during its extension import sanity checks. 1104 os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR, 1105 'libraries', 'usr', 'local', 'lib') 1106 print("Running configure...") 1107 runCommand("%s -C --enable-framework --enable-universalsdk=/ " 1108 "--with-universal-archs=%s " 1109 "%s " 1110 "%s " 1111 "%s " 1112 "%s " 1113 "LDFLAGS='-g -L%s/libraries/usr/local/lib' " 1114 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( 1115 shellQuote(os.path.join(SRCDIR, 'configure')), 1116 UNIVERSALARCHS, 1117 (' ', '--with-computed-gotos ')[PYTHON_3], 1118 (' ', '--without-ensurepip ')[PYTHON_3], 1119 (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( 1120 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1121 (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( 1122 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1123 shellQuote(WORKDIR)[1:-1], 1124 shellQuote(WORKDIR)[1:-1])) 1125 1126 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS 1127 # and, if defined, append its value to the make command. This allows 1128 # us to pass in version control tags, like GITTAG, to a build from a 1129 # tarball rather than from a vcs checkout, thus eliminating the need 1130 # to have a working copy of the vcs program on the build machine. 1131 # 1132 # A typical use might be: 1133 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ 1134 # GITVERSION='echo 123456789a' \ 1135 # GITTAG='echo v3.6.0' \ 1136 # GITBRANCH='echo 3.6'" 1137 1138 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") 1139 if make_extras: 1140 make_cmd = "make " + make_extras 1141 else: 1142 make_cmd = "make" 1143 print("Running " + make_cmd) 1144 runCommand(make_cmd) 1145 1146 print("Running make install") 1147 runCommand("make install DESTDIR=%s"%( 1148 shellQuote(rootDir))) 1149 1150 print("Running make frameworkinstallextras") 1151 runCommand("make frameworkinstallextras DESTDIR=%s"%( 1152 shellQuote(rootDir))) 1153 1154 del os.environ['DYLD_LIBRARY_PATH'] 1155 print("Copying required shared libraries") 1156 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): 1157 build_lib_dir = os.path.join( 1158 WORKDIR, 'libraries', 'Library', 'Frameworks', 1159 'Python.framework', 'Versions', getVersion(), 'lib') 1160 fw_lib_dir = os.path.join( 1161 WORKDIR, '_root', 'Library', 'Frameworks', 1162 'Python.framework', 'Versions', getVersion(), 'lib') 1163 if internalTk(): 1164 # move Tcl and Tk pkgconfig files 1165 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( 1166 shellQuote(build_lib_dir), 1167 shellQuote(fw_lib_dir) )) 1168 runCommand("rm -r %s/pkgconfig"%( 1169 shellQuote(build_lib_dir), )) 1170 runCommand("mv %s/* %s"%( 1171 shellQuote(build_lib_dir), 1172 shellQuote(fw_lib_dir) )) 1173 1174 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') 1175 frmDirVersioned = os.path.join(frmDir, 'Versions', version) 1176 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,)) 1177 # create directory for OpenSSL certificates 1178 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl') 1179 os.makedirs(sslDir) 1180 1181 print("Fix file modes") 1182 gid = grp.getgrnam('admin').gr_gid 1183 1184 shared_lib_error = False 1185 for dirpath, dirnames, filenames in os.walk(frmDir): 1186 for dn in dirnames: 1187 os.chmod(os.path.join(dirpath, dn), STAT_0o775) 1188 os.chown(os.path.join(dirpath, dn), -1, gid) 1189 1190 for fn in filenames: 1191 if os.path.islink(fn): 1192 continue 1193 1194 # "chmod g+w $fn" 1195 p = os.path.join(dirpath, fn) 1196 st = os.stat(p) 1197 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP) 1198 os.chown(p, -1, gid) 1199 1200 if fn in EXPECTED_SHARED_LIBS: 1201 # check to see that this file was linked with the 1202 # expected library path and version 1203 data = captureCommand("otool -L %s" % shellQuote(p)) 1204 for sl in EXPECTED_SHARED_LIBS[fn]: 1205 if ("\t%s " % sl) not in data: 1206 print("Expected shared lib %s was not linked with %s" 1207 % (sl, p)) 1208 shared_lib_error = True 1209 1210 if shared_lib_error: 1211 fatal("Unexpected shared library errors.") 1212 1213 if PYTHON_3: 1214 LDVERSION=None 1215 VERSION=None 1216 ABIFLAGS=None 1217 1218 fp = open(os.path.join(buildDir, 'Makefile'), 'r') 1219 for ln in fp: 1220 if ln.startswith('VERSION='): 1221 VERSION=ln.split()[1] 1222 if ln.startswith('ABIFLAGS='): 1223 ABIFLAGS=ln.split()[1] 1224 if ln.startswith('LDVERSION='): 1225 LDVERSION=ln.split()[1] 1226 fp.close() 1227 1228 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION) 1229 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS) 1230 config_suffix = '-' + LDVERSION 1231 if getVersionMajorMinor() >= (3, 6): 1232 config_suffix = config_suffix + '-darwin' 1233 else: 1234 config_suffix = '' # Python 2.x 1235 1236 # We added some directories to the search path during the configure 1237 # phase. Remove those because those directories won't be there on 1238 # the end-users system. Also remove the directories from _sysconfigdata.py 1239 # (added in 3.3) if it exists. 1240 1241 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,) 1242 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,) 1243 1244 # fix Makefile 1245 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile') 1246 fp = open(path, 'r') 1247 data = fp.read() 1248 fp.close() 1249 1250 for p in (include_path, lib_path): 1251 data = data.replace(" " + p, '') 1252 data = data.replace(p + " ", '') 1253 1254 fp = open(path, 'w') 1255 fp.write(data) 1256 fp.close() 1257 1258 # fix _sysconfigdata 1259 # 1260 # TODO: make this more robust! test_sysconfig_module of 1261 # distutils.tests.test_sysconfig.SysconfigTestCase tests that 1262 # the output from get_config_var in both sysconfig and 1263 # distutils.sysconfig is exactly the same for both CFLAGS and 1264 # LDFLAGS. The fixing up is now complicated by the pretty 1265 # printing in _sysconfigdata.py. Also, we are using the 1266 # pprint from the Python running the installer build which 1267 # may not cosmetically format the same as the pprint in the Python 1268 # being built (and which is used to originally generate 1269 # _sysconfigdata.py). 1270 1271 import pprint 1272 if getVersionMajorMinor() >= (3, 6): 1273 # XXX this is extra-fragile 1274 path = os.path.join(path_to_lib, '_sysconfigdata_m_darwin_darwin.py') 1275 else: 1276 path = os.path.join(path_to_lib, '_sysconfigdata.py') 1277 fp = open(path, 'r') 1278 data = fp.read() 1279 fp.close() 1280 # create build_time_vars dict 1281 exec(data) 1282 vars = {} 1283 for k, v in build_time_vars.items(): 1284 if type(v) == type(''): 1285 for p in (include_path, lib_path): 1286 v = v.replace(' ' + p, '') 1287 v = v.replace(p + ' ', '') 1288 vars[k] = v 1289 1290 fp = open(path, 'w') 1291 # duplicated from sysconfig._generate_posix_vars() 1292 fp.write('# system configuration generated and used by' 1293 ' the sysconfig module\n') 1294 fp.write('build_time_vars = ') 1295 pprint.pprint(vars, stream=fp) 1296 fp.close() 1297 1298 # Add symlinks in /usr/local/bin, using relative links 1299 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin') 1300 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks', 1301 'Python.framework', 'Versions', version, 'bin') 1302 if os.path.exists(usr_local_bin): 1303 shutil.rmtree(usr_local_bin) 1304 os.makedirs(usr_local_bin) 1305 for fn in os.listdir( 1306 os.path.join(frmDir, 'Versions', version, 'bin')): 1307 os.symlink(os.path.join(to_framework, fn), 1308 os.path.join(usr_local_bin, fn)) 1309 1310 os.chdir(curdir) 1311 1312 if PYTHON_3: 1313 # Remove the 'Current' link, that way we don't accidentally mess 1314 # with an already installed version of python 2 1315 os.unlink(os.path.join(rootDir, 'Library', 'Frameworks', 1316 'Python.framework', 'Versions', 'Current')) 1317 1318def patchFile(inPath, outPath): 1319 data = fileContents(inPath) 1320 data = data.replace('$FULL_VERSION', getFullVersion()) 1321 data = data.replace('$VERSION', getVersion()) 1322 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later'))) 1323 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS])) 1324 data = data.replace('$INSTALL_SIZE', installSize()) 1325 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS)) 1326 1327 # This one is not handy as a template variable 1328 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework') 1329 fp = open(outPath, 'w') 1330 fp.write(data) 1331 fp.close() 1332 1333def patchScript(inPath, outPath): 1334 major, minor = getVersionMajorMinor() 1335 data = fileContents(inPath) 1336 data = data.replace('@PYMAJOR@', str(major)) 1337 data = data.replace('@PYVER@', getVersion()) 1338 fp = open(outPath, 'w') 1339 fp.write(data) 1340 fp.close() 1341 os.chmod(outPath, STAT_0o755) 1342 1343 1344 1345def packageFromRecipe(targetDir, recipe): 1346 curdir = os.getcwd() 1347 try: 1348 # The major version (such as 2.5) is included in the package name 1349 # because having two version of python installed at the same time is 1350 # common. 1351 pkgname = '%s-%s'%(recipe['name'], getVersion()) 1352 srcdir = recipe.get('source') 1353 pkgroot = recipe.get('topdir', srcdir) 1354 postflight = recipe.get('postflight') 1355 readme = textwrap.dedent(recipe['readme']) 1356 isRequired = recipe.get('required', True) 1357 1358 print("- building package %s"%(pkgname,)) 1359 1360 # Substitute some variables 1361 textvars = dict( 1362 VER=getVersion(), 1363 FULLVER=getFullVersion(), 1364 ) 1365 readme = readme % textvars 1366 1367 if pkgroot is not None: 1368 pkgroot = pkgroot % textvars 1369 else: 1370 pkgroot = '/' 1371 1372 if srcdir is not None: 1373 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:]) 1374 srcdir = srcdir % textvars 1375 1376 if postflight is not None: 1377 postflight = os.path.abspath(postflight) 1378 1379 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents') 1380 os.makedirs(packageContents) 1381 1382 if srcdir is not None: 1383 os.chdir(srcdir) 1384 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1385 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1386 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),)) 1387 1388 fn = os.path.join(packageContents, 'PkgInfo') 1389 fp = open(fn, 'w') 1390 fp.write('pmkrpkg1') 1391 fp.close() 1392 1393 rsrcDir = os.path.join(packageContents, "Resources") 1394 os.mkdir(rsrcDir) 1395 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w') 1396 fp.write(readme) 1397 fp.close() 1398 1399 if postflight is not None: 1400 patchScript(postflight, os.path.join(rsrcDir, 'postflight')) 1401 1402 vers = getFullVersion() 1403 major, minor = getVersionMajorMinor() 1404 pl = Plist( 1405 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), 1406 CFBundleIdentifier='org.python.Python.%s'%(pkgname,), 1407 CFBundleName='Python.%s'%(pkgname,), 1408 CFBundleShortVersionString=vers, 1409 IFMajorVersion=major, 1410 IFMinorVersion=minor, 1411 IFPkgFormatVersion=0.10000000149011612, 1412 IFPkgFlagAllowBackRev=False, 1413 IFPkgFlagAuthorizationAction="RootAuthorization", 1414 IFPkgFlagDefaultLocation=pkgroot, 1415 IFPkgFlagFollowLinks=True, 1416 IFPkgFlagInstallFat=True, 1417 IFPkgFlagIsRequired=isRequired, 1418 IFPkgFlagOverwritePermissions=False, 1419 IFPkgFlagRelocatable=False, 1420 IFPkgFlagRestartAction="NoRestart", 1421 IFPkgFlagRootVolumeOnly=True, 1422 IFPkgFlagUpdateInstalledLangauges=False, 1423 ) 1424 writePlist(pl, os.path.join(packageContents, 'Info.plist')) 1425 1426 pl = Plist( 1427 IFPkgDescriptionDescription=readme, 1428 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), 1429 IFPkgDescriptionVersion=vers, 1430 ) 1431 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist')) 1432 1433 finally: 1434 os.chdir(curdir) 1435 1436 1437def makeMpkgPlist(path): 1438 1439 vers = getFullVersion() 1440 major, minor = getVersionMajorMinor() 1441 1442 pl = Plist( 1443 CFBundleGetInfoString="Python %s"%(vers,), 1444 CFBundleIdentifier='org.python.Python', 1445 CFBundleName='Python', 1446 CFBundleShortVersionString=vers, 1447 IFMajorVersion=major, 1448 IFMinorVersion=minor, 1449 IFPkgFlagComponentDirectory="Contents/Packages", 1450 IFPkgFlagPackageList=[ 1451 dict( 1452 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()), 1453 IFPkgFlagPackageSelection=item.get('selected', 'selected'), 1454 ) 1455 for item in pkg_recipes() 1456 ], 1457 IFPkgFormatVersion=0.10000000149011612, 1458 IFPkgFlagBackgroundScaling="proportional", 1459 IFPkgFlagBackgroundAlignment="left", 1460 IFPkgFlagAuthorizationAction="RootAuthorization", 1461 ) 1462 1463 writePlist(pl, path) 1464 1465 1466def buildInstaller(): 1467 1468 # Zap all compiled files 1469 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')): 1470 for fn in filenames: 1471 if fn.endswith('.pyc') or fn.endswith('.pyo'): 1472 os.unlink(os.path.join(dirpath, fn)) 1473 1474 outdir = os.path.join(WORKDIR, 'installer') 1475 if os.path.exists(outdir): 1476 shutil.rmtree(outdir) 1477 os.mkdir(outdir) 1478 1479 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents') 1480 pkgcontents = os.path.join(pkgroot, 'Packages') 1481 os.makedirs(pkgcontents) 1482 for recipe in pkg_recipes(): 1483 packageFromRecipe(pkgcontents, recipe) 1484 1485 rsrcDir = os.path.join(pkgroot, 'Resources') 1486 1487 fn = os.path.join(pkgroot, 'PkgInfo') 1488 fp = open(fn, 'w') 1489 fp.write('pmkrpkg1') 1490 fp.close() 1491 1492 os.mkdir(rsrcDir) 1493 1494 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) 1495 pl = Plist( 1496 IFPkgDescriptionTitle="Python", 1497 IFPkgDescriptionVersion=getVersion(), 1498 ) 1499 1500 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist')) 1501 for fn in os.listdir('resources'): 1502 if fn == '.svn': continue 1503 if fn.endswith('.jpg'): 1504 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1505 else: 1506 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1507 1508 1509def installSize(clear=False, _saved=[]): 1510 if clear: 1511 del _saved[:] 1512 if not _saved: 1513 data = captureCommand("du -ks %s"%( 1514 shellQuote(os.path.join(WORKDIR, '_root')))) 1515 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),)) 1516 return _saved[0] 1517 1518 1519def buildDMG(): 1520 """ 1521 Create DMG containing the rootDir. 1522 """ 1523 outdir = os.path.join(WORKDIR, 'diskimage') 1524 if os.path.exists(outdir): 1525 shutil.rmtree(outdir) 1526 1527 imagepath = os.path.join(outdir, 1528 'python-%s-macosx%s'%(getFullVersion(),DEPTARGET)) 1529 if INCLUDE_TIMESTAMP: 1530 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3]) 1531 imagepath = imagepath + '.dmg' 1532 1533 os.mkdir(outdir) 1534 volname='Python %s'%(getFullVersion()) 1535 runCommand("hdiutil create -format UDRW -volname %s -srcfolder %s %s"%( 1536 shellQuote(volname), 1537 shellQuote(os.path.join(WORKDIR, 'installer')), 1538 shellQuote(imagepath + ".tmp.dmg" ))) 1539 1540 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9, 1541 # when hdiutil fails with "Resource busy" 1542 1543 time.sleep(10) 1544 1545 if not os.path.exists(os.path.join(WORKDIR, "mnt")): 1546 os.mkdir(os.path.join(WORKDIR, "mnt")) 1547 runCommand("hdiutil attach %s -mountroot %s"%( 1548 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt")))) 1549 1550 # Custom icon for the DMG, shown when the DMG is mounted. 1551 shutil.copy("../Icons/Disk Image.icns", 1552 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns")) 1553 runCommand("SetFile -a C %s/"%( 1554 shellQuote(os.path.join(WORKDIR, "mnt", volname)),)) 1555 1556 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname)))) 1557 1558 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns") 1559 runCommand("hdiutil convert %s -format UDZO -o %s"%( 1560 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath))) 1561 setIcon(imagepath, "../Icons/Disk Image.icns") 1562 1563 os.unlink(imagepath + ".tmp.dmg") 1564 1565 return imagepath 1566 1567 1568def setIcon(filePath, icnsPath): 1569 """ 1570 Set the custom icon for the specified file or directory. 1571 """ 1572 1573 dirPath = os.path.normpath(os.path.dirname(__file__)) 1574 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon") 1575 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime: 1576 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due 1577 # to connections to the window server. 1578 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS") 1579 if not os.path.exists(appPath): 1580 os.makedirs(appPath) 1581 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%( 1582 shellQuote(toolPath), shellQuote(dirPath))) 1583 1584 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath), 1585 shellQuote(filePath))) 1586 1587def main(): 1588 # First parse options and check if we can perform our work 1589 parseOptions() 1590 checkEnvironment() 1591 1592 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET 1593 os.environ['CC'] = CC 1594 os.environ['CXX'] = CXX 1595 1596 if os.path.exists(WORKDIR): 1597 shutil.rmtree(WORKDIR) 1598 os.mkdir(WORKDIR) 1599 1600 os.environ['LC_ALL'] = 'C' 1601 1602 # Then build third-party libraries such as sleepycat DB4. 1603 buildLibraries() 1604 1605 # Now build python itself 1606 buildPython() 1607 1608 # And then build the documentation 1609 # Remove the Deployment Target from the shell 1610 # environment, it's no longer needed and 1611 # an unexpected build target can cause problems 1612 # when Sphinx and its dependencies need to 1613 # be (re-)installed. 1614 del os.environ['MACOSX_DEPLOYMENT_TARGET'] 1615 buildPythonDocs() 1616 1617 1618 # Prepare the applications folder 1619 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%( 1620 getVersion(),)) 1621 fn = os.path.join(folder, "License.rtf") 1622 patchFile("resources/License.rtf", fn) 1623 fn = os.path.join(folder, "ReadMe.rtf") 1624 patchFile("resources/ReadMe.rtf", fn) 1625 fn = os.path.join(folder, "Update Shell Profile.command") 1626 patchScript("scripts/postflight.patch-profile", fn) 1627 fn = os.path.join(folder, "Install Certificates.command") 1628 patchScript("resources/install_certificates.command", fn) 1629 os.chmod(folder, STAT_0o755) 1630 setIcon(folder, "../Icons/Python Folder.icns") 1631 1632 # Create the installer 1633 buildInstaller() 1634 1635 # And copy the readme into the directory containing the installer 1636 patchFile('resources/ReadMe.rtf', 1637 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf')) 1638 1639 # Ditto for the license file. 1640 patchFile('resources/License.rtf', 1641 os.path.join(WORKDIR, 'installer', 'License.rtf')) 1642 1643 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w') 1644 fp.write("# BUILD INFO\n") 1645 fp.write("# Date: %s\n" % time.ctime()) 1646 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos) 1647 fp.close() 1648 1649 # And copy it to a DMG 1650 buildDMG() 1651 1652if __name__ == "__main__": 1653 main() 1654