1#!/usr/bin/env python 2""" 3This script is used to build "official" universal installers on macOS. 4 5NEW for 3.9.0 and backports: 6- 2.7 end-of-life issues: 7 - Python 3 installs now update the Current version link 8 in /Library/Frameworks/Python.framework/Versions 9- fully support running under Python 3 as well as 2.7 10- support building on newer macOS systems with SIP 11- fully support building on macOS 10.9+ 12- support 10.6+ on best effort 13- support bypassing docs build by supplying a prebuilt 14 docs html tarball in the third-party source library, 15 in the format and filename conventional of those 16 downloadable from python.org: 17 python-3.x.y-docs-html.tar.bz2 18 19NEW for 3.7.0: 20- support Intel 64-bit-only () and 32-bit-only installer builds 21- build and use internal Tcl/Tk 8.6 for 10.6+ builds 22- deprecate use of explicit SDK (--sdk-path=) since all but the oldest 23 versions of Xcode support implicit setting of an SDK via environment 24 variables (SDKROOT and friends, see the xcrun man page for more info). 25 The SDK stuff was primarily needed for building universal installers 26 for 10.4; so as of 3.7.0, building installers for 10.4 is no longer 27 supported with build-installer. 28- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" 29 30TODO: 31- test building with SDKROOT and DEVELOPER_DIR xcrun env variables 32 33Usage: see USAGE variable in the script. 34""" 35import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp 36try: 37 import urllib2 as urllib_request 38except ImportError: 39 import urllib.request as urllib_request 40 41STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 42 | stat.S_IRGRP | stat.S_IXGRP 43 | stat.S_IROTH | stat.S_IXOTH ) 44 45STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 46 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP 47 | stat.S_IROTH | stat.S_IXOTH ) 48 49INCLUDE_TIMESTAMP = 1 50VERBOSE = 1 51 52RUNNING_ON_PYTHON2 = sys.version_info.major == 2 53 54if RUNNING_ON_PYTHON2: 55 from plistlib import writePlist 56else: 57 from plistlib import dump 58 def writePlist(path, plist): 59 with open(plist, 'wb') as fp: 60 dump(path, fp) 61 62def shellQuote(value): 63 """ 64 Return the string value in a form that can safely be inserted into 65 a shell command. 66 """ 67 return "'%s'"%(value.replace("'", "'\"'\"'")) 68 69def grepValue(fn, variable): 70 """ 71 Return the unquoted value of a variable from a file.. 72 QUOTED_VALUE='quotes' -> str('quotes') 73 UNQUOTED_VALUE=noquotes -> str('noquotes') 74 """ 75 variable = variable + '=' 76 for ln in open(fn, 'r'): 77 if ln.startswith(variable): 78 value = ln[len(variable):].strip() 79 return value.strip("\"'") 80 raise RuntimeError("Cannot find variable %s" % variable[:-1]) 81 82_cache_getVersion = None 83 84def getVersion(): 85 global _cache_getVersion 86 if _cache_getVersion is None: 87 _cache_getVersion = grepValue( 88 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION') 89 return _cache_getVersion 90 91def getVersionMajorMinor(): 92 return tuple([int(n) for n in getVersion().split('.', 2)]) 93 94_cache_getFullVersion = None 95 96def getFullVersion(): 97 global _cache_getFullVersion 98 if _cache_getFullVersion is not None: 99 return _cache_getFullVersion 100 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h') 101 for ln in open(fn): 102 if 'PY_VERSION' in ln: 103 _cache_getFullVersion = ln.split()[-1][1:-1] 104 return _cache_getFullVersion 105 raise RuntimeError("Cannot find full version??") 106 107FW_PREFIX = ["Library", "Frameworks", "Python.framework"] 108FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions 109FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions 110 111# The directory we'll use to create the build (will be erased and recreated) 112WORKDIR = "/tmp/_py" 113 114# The directory we'll use to store third-party sources. Set this to something 115# else if you don't want to re-fetch required libraries every time. 116DEPSRC = os.path.join(WORKDIR, 'third-party') 117DEPSRC = os.path.expanduser('~/Universal/other-sources') 118 119universal_opts_map = { '32-bit': ('i386', 'ppc',), 120 '64-bit': ('x86_64', 'ppc64',), 121 'intel': ('i386', 'x86_64'), 122 'intel-32': ('i386',), 123 'intel-64': ('x86_64',), 124 '3-way': ('ppc', 'i386', 'x86_64'), 125 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } 126default_target_map = { 127 '64-bit': '10.5', 128 '3-way': '10.5', 129 'intel': '10.5', 130 'intel-32': '10.4', 131 'intel-64': '10.5', 132 'all': '10.5', 133} 134 135UNIVERSALOPTS = tuple(universal_opts_map.keys()) 136 137UNIVERSALARCHS = '32-bit' 138 139ARCHLIST = universal_opts_map[UNIVERSALARCHS] 140 141# Source directory (assume we're in Mac/BuildScript) 142SRCDIR = os.path.dirname( 143 os.path.dirname( 144 os.path.dirname( 145 os.path.abspath(__file__ 146 )))) 147 148# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level 149DEPTARGET = '10.5' 150 151def getDeptargetTuple(): 152 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) 153 154def getTargetCompilers(): 155 target_cc_map = { 156 '10.4': ('gcc-4.0', 'g++-4.0'), 157 '10.5': ('gcc', 'g++'), 158 '10.6': ('gcc', 'g++'), 159 } 160 return target_cc_map.get(DEPTARGET, ('gcc', 'g++') ) 161 162CC, CXX = getTargetCompilers() 163 164PYTHON_3 = getVersionMajorMinor() >= (3, 0) 165 166USAGE = textwrap.dedent("""\ 167 Usage: build_python [options] 168 169 Options: 170 -? or -h: Show this message 171 -b DIR 172 --build-dir=DIR: Create build here (default: %(WORKDIR)r) 173 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) 174 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) 175 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) 176 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) 177 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) 178""")% globals() 179 180# Dict of object file names with shared library names to check after building. 181# This is to ensure that we ended up dynamically linking with the shared 182# library paths and versions we expected. For example: 183# EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 184# '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl', 185# '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] 186EXPECTED_SHARED_LIBS = {} 187 188# Are we building and linking with our own copy of Tcl/TK? 189# For now, do so if deployment target is 10.6+. 190def internalTk(): 191 return getDeptargetTuple() >= (10, 6) 192 193# List of names of third party software built with this installer. 194# The names will be inserted into the rtf version of the License. 195THIRD_PARTY_LIBS = [] 196 197# Instructions for building libraries that are necessary for building a 198# batteries included python. 199# [The recipes are defined here for convenience but instantiated later after 200# command line options have been processed.] 201def library_recipes(): 202 result = [] 203 204 LT_10_5 = bool(getDeptargetTuple() < (10, 5)) 205 206 # Since Apple removed the header files for the deprecated system 207 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not 208 # have much choice but to build our own copy here, too. 209 210 result.extend([ 211 dict( 212 name="OpenSSL 1.1.1g", 213 url="https://www.openssl.org/source/openssl-1.1.1g.tar.gz", 214 checksum='76766e98997660138cdaf13a187bd234', 215 buildrecipe=build_universal_openssl, 216 configure=None, 217 install=None, 218 ), 219 ]) 220 221 if internalTk(): 222 result.extend([ 223 dict( 224 name="Tcl 8.6.8", 225 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz", 226 checksum='81656d3367af032e0ae6157eff134f89', 227 buildDir="unix", 228 configure_pre=[ 229 '--enable-shared', 230 '--enable-threads', 231 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 232 ], 233 useLDFlags=False, 234 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 235 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 236 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 237 }, 238 ), 239 dict( 240 name="Tk 8.6.8", 241 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz", 242 checksum='5e0faecba458ee1386078fb228d008ba', 243 patches=[ 244 "tk868_on_10_8_10_9.patch", 245 ], 246 buildDir="unix", 247 configure_pre=[ 248 '--enable-aqua', 249 '--enable-shared', 250 '--enable-threads', 251 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 252 ], 253 useLDFlags=False, 254 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 255 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 256 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 257 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), 258 }, 259 ), 260 ]) 261 262 if PYTHON_3: 263 result.extend([ 264 dict( 265 name="XZ 5.2.3", 266 url="http://tukaani.org/xz/xz-5.2.3.tar.gz", 267 checksum='ef68674fb47a8b8e741b34e429d86e9d', 268 configure_pre=[ 269 '--disable-dependency-tracking', 270 ] 271 ), 272 ]) 273 274 result.extend([ 275 dict( 276 name="NCurses 5.9", 277 url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz", 278 checksum='8cb9c412e5f2d96bc6f459aa8c6282a1', 279 configure_pre=[ 280 "--enable-widec", 281 "--without-cxx", 282 "--without-cxx-binding", 283 "--without-ada", 284 "--without-curses-h", 285 "--enable-shared", 286 "--with-shared", 287 "--without-debug", 288 "--without-normal", 289 "--without-tests", 290 "--without-manpages", 291 "--datadir=/usr/share", 292 "--sysconfdir=/etc", 293 "--sharedstatedir=/usr/com", 294 "--with-terminfo-dirs=/usr/share/terminfo", 295 "--with-default-terminfo-dir=/usr/share/terminfo", 296 "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), 297 ], 298 patchscripts=[ 299 ("ftp://ftp.invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", 300 "f54bf02a349f96a7c4f0d00922f3a0d4"), 301 ], 302 useLDFlags=False, 303 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%( 304 shellQuote(os.path.join(WORKDIR, 'libraries')), 305 shellQuote(os.path.join(WORKDIR, 'libraries')), 306 getVersion(), 307 ), 308 ), 309 dict( 310 name="SQLite 3.32.3", 311 url="https://sqlite.org/2020/sqlite-autoconf-3320300.tar.gz", 312 checksum='2e3911a3c15e85c2f2d040154bbe5ce3', 313 extra_cflags=('-Os ' 314 '-DSQLITE_ENABLE_FTS5 ' 315 '-DSQLITE_ENABLE_FTS4 ' 316 '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' 317 '-DSQLITE_ENABLE_JSON1 ' 318 '-DSQLITE_ENABLE_RTREE ' 319 '-DSQLITE_TCL=0 ' 320 '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), 321 configure_pre=[ 322 '--enable-threadsafe', 323 '--enable-shared=no', 324 '--enable-static=yes', 325 '--disable-readline', 326 '--disable-dependency-tracking', 327 ] 328 ), 329 ]) 330 331 if getDeptargetTuple() < (10, 5): 332 result.extend([ 333 dict( 334 name="Bzip2 1.0.6", 335 url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", 336 checksum='00b516f4704d4a7cb50a1d97e6e8e15b', 337 configure=None, 338 install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( 339 CC, CXX, 340 shellQuote(os.path.join(WORKDIR, 'libraries')), 341 ' -arch '.join(ARCHLIST), 342 ), 343 ), 344 dict( 345 name="ZLib 1.2.3", 346 url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", 347 checksum='debc62758716a169df9f62e6ab2bc634', 348 configure=None, 349 install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( 350 CC, CXX, 351 shellQuote(os.path.join(WORKDIR, 'libraries')), 352 ' -arch '.join(ARCHLIST), 353 ), 354 ), 355 dict( 356 # Note that GNU readline is GPL'd software 357 name="GNU Readline 6.1.2", 358 url="http://ftp.gnu.org/pub/gnu/readline/readline-6.1.tar.gz" , 359 checksum='fc2f7e714fe792db1ce6ddc4c9fb4ef3', 360 patchlevel='0', 361 patches=[ 362 # The readline maintainers don't do actual micro releases, but 363 # just ship a set of patches. 364 ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-001', 365 'c642f2e84d820884b0bf9fd176bc6c3f'), 366 ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-002', 367 '1a76781a1ea734e831588285db7ec9b1'), 368 ] 369 ), 370 ]) 371 372 if not PYTHON_3: 373 result.extend([ 374 dict( 375 name="Sleepycat DB 4.7.25", 376 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz", 377 checksum='ec2b87e833779681a0c3a814aa71359e', 378 buildDir="build_unix", 379 configure="../dist/configure", 380 configure_pre=[ 381 '--includedir=/usr/local/include/db4', 382 ] 383 ), 384 ]) 385 386 return result 387 388 389# Instructions for building packages inside the .mpkg. 390def pkg_recipes(): 391 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3] 392 result = [ 393 dict( 394 name="PythonFramework", 395 long_name="Python Framework", 396 source="/Library/Frameworks/Python.framework", 397 readme="""\ 398 This package installs Python.framework, that is the python 399 interpreter and the standard library. 400 """, 401 postflight="scripts/postflight.framework", 402 selected='selected', 403 ), 404 dict( 405 name="PythonApplications", 406 long_name="GUI Applications", 407 source="/Applications/Python %(VER)s", 408 readme="""\ 409 This package installs IDLE (an interactive Python IDE), 410 Python Launcher and Build Applet (create application bundles 411 from python scripts). 412 413 It also installs a number of examples and demos. 414 """, 415 required=False, 416 selected='selected', 417 ), 418 dict( 419 name="PythonUnixTools", 420 long_name="UNIX command-line tools", 421 source="/usr/local/bin", 422 readme="""\ 423 This package installs the unix tools in /usr/local/bin for 424 compatibility with older releases of Python. This package 425 is not necessary to use Python. 426 """, 427 required=False, 428 selected='selected', 429 ), 430 dict( 431 name="PythonDocumentation", 432 long_name="Python Documentation", 433 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation", 434 source="/pydocs", 435 readme="""\ 436 This package installs the python documentation at a location 437 that is useable for pydoc and IDLE. 438 """, 439 postflight="scripts/postflight.documentation", 440 required=False, 441 selected='selected', 442 ), 443 dict( 444 name="PythonProfileChanges", 445 long_name="Shell profile updater", 446 readme="""\ 447 This packages updates your shell profile to make sure that 448 the Python tools are found by your shell in preference of 449 the system provided Python tools. 450 451 If you don't install this package you'll have to add 452 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin" 453 to your PATH by hand. 454 """, 455 postflight="scripts/postflight.patch-profile", 456 topdir="/Library/Frameworks/Python.framework", 457 source="/empty-dir", 458 required=False, 459 selected='selected', 460 ), 461 dict( 462 name="PythonInstallPip", 463 long_name="Install or upgrade pip", 464 readme="""\ 465 This package installs (or upgrades from an earlier version) 466 pip, a tool for installing and managing Python packages. 467 """, 468 postflight="scripts/postflight.ensurepip", 469 topdir="/Library/Frameworks/Python.framework", 470 source="/empty-dir", 471 required=False, 472 selected='selected', 473 ), 474 ] 475 476 return result 477 478def fatal(msg): 479 """ 480 A fatal error, bail out. 481 """ 482 sys.stderr.write('FATAL: ') 483 sys.stderr.write(msg) 484 sys.stderr.write('\n') 485 sys.exit(1) 486 487def fileContents(fn): 488 """ 489 Return the contents of the named file 490 """ 491 return open(fn, 'r').read() 492 493def runCommand(commandline): 494 """ 495 Run a command and raise RuntimeError if it fails. Output is suppressed 496 unless the command fails. 497 """ 498 fd = os.popen(commandline, 'r') 499 data = fd.read() 500 xit = fd.close() 501 if xit is not None: 502 sys.stdout.write(data) 503 raise RuntimeError("command failed: %s"%(commandline,)) 504 505 if VERBOSE: 506 sys.stdout.write(data); sys.stdout.flush() 507 508def captureCommand(commandline): 509 fd = os.popen(commandline, 'r') 510 data = fd.read() 511 xit = fd.close() 512 if xit is not None: 513 sys.stdout.write(data) 514 raise RuntimeError("command failed: %s"%(commandline,)) 515 516 return data 517 518def getTclTkVersion(configfile, versionline): 519 """ 520 search Tcl or Tk configuration file for version line 521 """ 522 try: 523 f = open(configfile, "r") 524 except OSError: 525 fatal("Framework configuration file not found: %s" % configfile) 526 527 for l in f: 528 if l.startswith(versionline): 529 f.close() 530 return l 531 532 fatal("Version variable %s not found in framework configuration file: %s" 533 % (versionline, configfile)) 534 535def checkEnvironment(): 536 """ 537 Check that we're running on a supported system. 538 """ 539 540 if sys.version_info[0:2] < (2, 5): 541 fatal("This script must be run with Python 2.5 (or later)") 542 543 if platform.system() != 'Darwin': 544 fatal("This script should be run on a macOS 10.5 (or later) system") 545 546 if int(platform.release().split('.')[0]) < 8: 547 fatal("This script should be run on a macOS 10.5 (or later) system") 548 549 # Because we only support dynamic load of only one major/minor version of 550 # Tcl/Tk, if we are not using building and using our own private copy of 551 # Tcl/Tk, ensure: 552 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked 553 # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer 554 # enforce that the version of the user-installed framework also 555 # exists in the system-supplied Tcl/Tk frameworks. Time to support 556 # Tcl/Tk 8.6 even if Apple does not. 557 if not internalTk(): 558 frameworks = {} 559 for framework in ['Tcl', 'Tk']: 560 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework 561 libfw = os.path.join('/', fwpth) 562 usrfw = os.path.join(os.getenv('HOME'), fwpth) 563 frameworks[framework] = os.readlink(libfw) 564 if not os.path.exists(libfw): 565 fatal("Please install a link to a current %s %s as %s so " 566 "the user can override the system framework." 567 % (framework, frameworks[framework], libfw)) 568 if os.path.exists(usrfw): 569 fatal("Please rename %s to avoid possible dynamic load issues." 570 % usrfw) 571 572 if frameworks['Tcl'] != frameworks['Tk']: 573 fatal("The Tcl and Tk frameworks are not the same version.") 574 575 print(" -- Building with external Tcl/Tk %s frameworks" 576 % frameworks['Tk']) 577 578 # add files to check after build 579 EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 580 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" 581 % frameworks['Tcl'], 582 "/Library/Frameworks/Tk.framework/Versions/%s/Tk" 583 % frameworks['Tk'], 584 ] 585 else: 586 print(" -- Building private copy of Tcl/Tk") 587 print("") 588 589 # Remove inherited environment variables which might influence build 590 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 591 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] 592 for ev in list(os.environ): 593 for prefix in environ_var_prefixes: 594 if ev.startswith(prefix) : 595 print("INFO: deleting environment variable %s=%s" % ( 596 ev, os.environ[ev])) 597 del os.environ[ev] 598 599 base_path = '/bin:/sbin:/usr/bin:/usr/sbin' 600 if 'SDK_TOOLS_BIN' in os.environ: 601 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path 602 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; 603 # add its fixed location here if it exists 604 OLD_DEVELOPER_TOOLS = '/Developer/Tools' 605 if os.path.isdir(OLD_DEVELOPER_TOOLS): 606 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS 607 os.environ['PATH'] = base_path 608 print("Setting default PATH: %s"%(os.environ['PATH'])) 609 # Ensure we have access to sphinx-build. 610 # You may have to create a link in /usr/bin for it. 611 runCommand('sphinx-build --version') 612 613def parseOptions(args=None): 614 """ 615 Parse arguments and update global settings. 616 """ 617 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET 618 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX 619 global FW_VERSION_PREFIX 620 global FW_SSL_DIRECTORY 621 622 if args is None: 623 args = sys.argv[1:] 624 625 try: 626 options, args = getopt.getopt(args, '?hb', 627 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', 628 'dep-target=', 'universal-archs=', 'help' ]) 629 except getopt.GetoptError: 630 print(sys.exc_info()[1]) 631 sys.exit(1) 632 633 if args: 634 print("Additional arguments") 635 sys.exit(1) 636 637 deptarget = None 638 for k, v in options: 639 if k in ('-h', '-?', '--help'): 640 print(USAGE) 641 sys.exit(0) 642 643 elif k in ('-d', '--build-dir'): 644 WORKDIR=v 645 646 elif k in ('--third-party',): 647 DEPSRC=v 648 649 elif k in ('--sdk-path',): 650 print(" WARNING: --sdk-path is no longer supported") 651 652 elif k in ('--src-dir',): 653 SRCDIR=v 654 655 elif k in ('--dep-target', ): 656 DEPTARGET=v 657 deptarget=v 658 659 elif k in ('--universal-archs', ): 660 if v in UNIVERSALOPTS: 661 UNIVERSALARCHS = v 662 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 663 if deptarget is None: 664 # Select alternate default deployment 665 # target 666 DEPTARGET = default_target_map.get(v, '10.5') 667 else: 668 raise NotImplementedError(v) 669 670 else: 671 raise NotImplementedError(k) 672 673 SRCDIR=os.path.abspath(SRCDIR) 674 WORKDIR=os.path.abspath(WORKDIR) 675 DEPSRC=os.path.abspath(DEPSRC) 676 677 CC, CXX = getTargetCompilers() 678 679 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()] 680 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"] 681 682 print("-- Settings:") 683 print(" * Source directory: %s" % SRCDIR) 684 print(" * Build directory: %s" % WORKDIR) 685 print(" * Third-party source: %s" % DEPSRC) 686 print(" * Deployment target: %s" % DEPTARGET) 687 print(" * Universal archs: %s" % str(ARCHLIST)) 688 print(" * C compiler: %s" % CC) 689 print(" * C++ compiler: %s" % CXX) 690 print("") 691 print(" -- Building a Python %s framework at patch level %s" 692 % (getVersion(), getFullVersion())) 693 print("") 694 695def extractArchive(builddir, archiveName): 696 """ 697 Extract a source archive into 'builddir'. Returns the path of the 698 extracted archive. 699 700 XXX: This function assumes that archives contain a toplevel directory 701 that is has the same name as the basename of the archive. This is 702 safe enough for almost anything we use. Unfortunately, it does not 703 work for current Tcl and Tk source releases where the basename of 704 the archive ends with "-src" but the uncompressed directory does not. 705 For now, just special case Tcl and Tk tar.gz downloads. 706 """ 707 curdir = os.getcwd() 708 try: 709 os.chdir(builddir) 710 if archiveName.endswith('.tar.gz'): 711 retval = os.path.basename(archiveName[:-7]) 712 if ((retval.startswith('tcl') or retval.startswith('tk')) 713 and retval.endswith('-src')): 714 retval = retval[:-4] 715 if os.path.exists(retval): 716 shutil.rmtree(retval) 717 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') 718 719 elif archiveName.endswith('.tar.bz2'): 720 retval = os.path.basename(archiveName[:-8]) 721 if os.path.exists(retval): 722 shutil.rmtree(retval) 723 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r') 724 725 elif archiveName.endswith('.tar'): 726 retval = os.path.basename(archiveName[:-4]) 727 if os.path.exists(retval): 728 shutil.rmtree(retval) 729 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r') 730 731 elif archiveName.endswith('.zip'): 732 retval = os.path.basename(archiveName[:-4]) 733 if os.path.exists(retval): 734 shutil.rmtree(retval) 735 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r') 736 737 data = fp.read() 738 xit = fp.close() 739 if xit is not None: 740 sys.stdout.write(data) 741 raise RuntimeError("Cannot extract %s"%(archiveName,)) 742 743 return os.path.join(builddir, retval) 744 745 finally: 746 os.chdir(curdir) 747 748def downloadURL(url, fname): 749 """ 750 Download the contents of the url into the file. 751 """ 752 fpIn = urllib_request.urlopen(url) 753 fpOut = open(fname, 'wb') 754 block = fpIn.read(10240) 755 try: 756 while block: 757 fpOut.write(block) 758 block = fpIn.read(10240) 759 fpIn.close() 760 fpOut.close() 761 except: 762 try: 763 os.unlink(fname) 764 except OSError: 765 pass 766 767def verifyThirdPartyFile(url, checksum, fname): 768 """ 769 Download file from url to filename fname if it does not already exist. 770 Abort if file contents does not match supplied md5 checksum. 771 """ 772 name = os.path.basename(fname) 773 if os.path.exists(fname): 774 print("Using local copy of %s"%(name,)) 775 else: 776 print("Did not find local copy of %s"%(name,)) 777 print("Downloading %s"%(name,)) 778 downloadURL(url, fname) 779 print("Archive for %s stored as %s"%(name, fname)) 780 if os.system( 781 'MD5=$(openssl md5 %s) ; test "${MD5##*= }" = "%s"' 782 % (shellQuote(fname), checksum) ): 783 fatal('MD5 checksum mismatch for file %s' % fname) 784 785def build_universal_openssl(basedir, archList): 786 """ 787 Special case build recipe for universal build of openssl. 788 789 The upstream OpenSSL build system does not directly support 790 OS X universal builds. We need to build each architecture 791 separately then lipo them together into fat libraries. 792 """ 793 794 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4). 795 # If we are building on a 10.4.x or earlier system, 796 # unilaterally disable assembly code building to avoid the problem. 797 no_asm = int(platform.release().split(".")[0]) < 9 798 799 def build_openssl_arch(archbase, arch): 800 "Build one architecture of openssl" 801 arch_opts = { 802 "i386": ["darwin-i386-cc"], 803 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], 804 "ppc": ["darwin-ppc-cc"], 805 "ppc64": ["darwin64-ppc-cc"], 806 } 807 808 # Somewhere between OpenSSL 1.1.0j and 1.1.1c, changes cause the 809 # "enable-ec_nistp_64_gcc_128" option to get compile errors when 810 # building on our 10.6 gcc-4.2 environment. There have been other 811 # reports of projects running into this when using older compilers. 812 # So, for now, do not try to use "enable-ec_nistp_64_gcc_128" when 813 # building for 10.6. 814 if getDeptargetTuple() == (10, 6): 815 arch_opts['x86_64'].remove('enable-ec_nistp_64_gcc_128') 816 817 configure_opts = [ 818 "no-idea", 819 "no-mdc2", 820 "no-rc5", 821 "no-zlib", 822 "no-ssl3", 823 # "enable-unit-test", 824 "shared", 825 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), 826 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), 827 ] 828 if no_asm: 829 configure_opts.append("no-asm") 830 runCommand(" ".join(["perl", "Configure"] 831 + arch_opts[arch] + configure_opts)) 832 runCommand("make depend") 833 runCommand("make all") 834 runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase)) 835 # runCommand("make test") 836 return 837 838 srcdir = os.getcwd() 839 universalbase = os.path.join(srcdir, "..", 840 os.path.basename(srcdir) + "-universal") 841 os.mkdir(universalbase) 842 archbasefws = [] 843 for arch in archList: 844 # fresh copy of the source tree 845 archsrc = os.path.join(universalbase, arch, "src") 846 shutil.copytree(srcdir, archsrc, symlinks=True) 847 # install base for this arch 848 archbase = os.path.join(universalbase, arch, "root") 849 os.mkdir(archbase) 850 # Python framework base within install_prefix: 851 # the build will install into this framework.. 852 # This is to ensure that the resulting shared libs have 853 # the desired real install paths built into them. 854 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX) 855 856 # build one architecture 857 os.chdir(archsrc) 858 build_openssl_arch(archbase, arch) 859 os.chdir(srcdir) 860 archbasefws.append(archbasefw) 861 862 # copy arch-independent files from last build into the basedir framework 863 basefw = os.path.join(basedir, *FW_VERSION_PREFIX) 864 shutil.copytree( 865 os.path.join(archbasefw, "include", "openssl"), 866 os.path.join(basefw, "include", "openssl") 867 ) 868 869 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"), 870 "SHLIB_VERSION_NUMBER") 871 # e.g. -> "1.0.0" 872 libcrypto = "libcrypto.dylib" 873 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".") 874 # e.g. -> "libcrypto.1.0.0.dylib" 875 libssl = "libssl.dylib" 876 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".") 877 # e.g. -> "libssl.1.0.0.dylib" 878 879 try: 880 os.mkdir(os.path.join(basefw, "lib")) 881 except OSError: 882 pass 883 884 # merge the individual arch-dependent shared libs into a fat shared lib 885 archbasefws.insert(0, basefw) 886 for (lib_unversioned, lib_versioned) in [ 887 (libcrypto, libcrypto_versioned), 888 (libssl, libssl_versioned) 889 ]: 890 runCommand("lipo -create -output " + 891 " ".join(shellQuote( 892 os.path.join(fw, "lib", lib_versioned)) 893 for fw in archbasefws)) 894 # and create an unversioned symlink of it 895 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned)) 896 897 # Create links in the temp include and lib dirs that will be injected 898 # into the Python build so that setup.py can find them while building 899 # and the versioned links so that the setup.py post-build import test 900 # does not fail. 901 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX) 902 for fn in [ 903 ["include", "openssl"], 904 ["lib", libcrypto], 905 ["lib", libssl], 906 ["lib", libcrypto_versioned], 907 ["lib", libssl_versioned], 908 ]: 909 os.symlink( 910 os.path.join(relative_path, *fn), 911 os.path.join(basedir, "usr", "local", *fn) 912 ) 913 914 return 915 916def buildRecipe(recipe, basedir, archList): 917 """ 918 Build software using a recipe. This function does the 919 'configure;make;make install' dance for C software, with a possibility 920 to customize this process, basically a poor-mans DarwinPorts. 921 """ 922 curdir = os.getcwd() 923 924 name = recipe['name'] 925 THIRD_PARTY_LIBS.append(name) 926 url = recipe['url'] 927 configure = recipe.get('configure', './configure') 928 buildrecipe = recipe.get('buildrecipe', None) 929 install = recipe.get('install', 'make && make install DESTDIR=%s'%( 930 shellQuote(basedir))) 931 932 archiveName = os.path.split(url)[-1] 933 sourceArchive = os.path.join(DEPSRC, archiveName) 934 935 if not os.path.exists(DEPSRC): 936 os.mkdir(DEPSRC) 937 938 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive) 939 print("Extracting archive for %s"%(name,)) 940 buildDir=os.path.join(WORKDIR, '_bld') 941 if not os.path.exists(buildDir): 942 os.mkdir(buildDir) 943 944 workDir = extractArchive(buildDir, sourceArchive) 945 os.chdir(workDir) 946 947 for patch in recipe.get('patches', ()): 948 if isinstance(patch, tuple): 949 url, checksum = patch 950 fn = os.path.join(DEPSRC, os.path.basename(url)) 951 verifyThirdPartyFile(url, checksum, fn) 952 else: 953 # patch is a file in the source directory 954 fn = os.path.join(curdir, patch) 955 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1), 956 shellQuote(fn),)) 957 958 for patchscript in recipe.get('patchscripts', ()): 959 if isinstance(patchscript, tuple): 960 url, checksum = patchscript 961 fn = os.path.join(DEPSRC, os.path.basename(url)) 962 verifyThirdPartyFile(url, checksum, fn) 963 else: 964 # patch is a file in the source directory 965 fn = os.path.join(curdir, patchscript) 966 if fn.endswith('.bz2'): 967 runCommand('bunzip2 -fk %s' % shellQuote(fn)) 968 fn = fn[:-4] 969 runCommand('sh %s' % shellQuote(fn)) 970 os.unlink(fn) 971 972 if 'buildDir' in recipe: 973 os.chdir(recipe['buildDir']) 974 975 if configure is not None: 976 configure_args = [ 977 "--prefix=/usr/local", 978 "--enable-static", 979 "--disable-shared", 980 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),), 981 ] 982 983 if 'configure_pre' in recipe: 984 args = list(recipe['configure_pre']) 985 if '--disable-static' in args: 986 configure_args.remove('--enable-static') 987 if '--enable-shared' in args: 988 configure_args.remove('--disable-shared') 989 configure_args.extend(args) 990 991 if recipe.get('useLDFlags', 1): 992 configure_args.extend([ 993 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 994 "-I%s/usr/local/include"%( 995 recipe.get('extra_cflags', ''), 996 DEPTARGET, 997 ' -arch '.join(archList), 998 shellQuote(basedir)[1:-1],), 999 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( 1000 DEPTARGET, 1001 shellQuote(basedir)[1:-1], 1002 ' -arch '.join(archList)), 1003 ]) 1004 else: 1005 configure_args.extend([ 1006 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1007 "-I%s/usr/local/include"%( 1008 recipe.get('extra_cflags', ''), 1009 DEPTARGET, 1010 ' -arch '.join(archList), 1011 shellQuote(basedir)[1:-1],), 1012 ]) 1013 1014 if 'configure_post' in recipe: 1015 configure_args = configure_args + list(recipe['configure_post']) 1016 1017 configure_args.insert(0, configure) 1018 configure_args = [ shellQuote(a) for a in configure_args ] 1019 1020 print("Running configure for %s"%(name,)) 1021 runCommand(' '.join(configure_args) + ' 2>&1') 1022 1023 if buildrecipe is not None: 1024 # call special-case build recipe, e.g. for openssl 1025 buildrecipe(basedir, archList) 1026 1027 if install is not None: 1028 print("Running install for %s"%(name,)) 1029 runCommand('{ ' + install + ' ;} 2>&1') 1030 1031 print("Done %s"%(name,)) 1032 print("") 1033 1034 os.chdir(curdir) 1035 1036def buildLibraries(): 1037 """ 1038 Build our dependencies into $WORKDIR/libraries/usr/local 1039 """ 1040 print("") 1041 print("Building required libraries") 1042 print("") 1043 universal = os.path.join(WORKDIR, 'libraries') 1044 os.mkdir(universal) 1045 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib')) 1046 os.makedirs(os.path.join(universal, 'usr', 'local', 'include')) 1047 1048 for recipe in library_recipes(): 1049 buildRecipe(recipe, universal, ARCHLIST) 1050 1051 1052 1053def buildPythonDocs(): 1054 # This stores the documentation as Resources/English.lproj/Documentation 1055 # inside the framework. pydoc and IDLE will pick it up there. 1056 print("Install python documentation") 1057 rootDir = os.path.join(WORKDIR, '_root') 1058 buildDir = os.path.join('../../Doc') 1059 docdir = os.path.join(rootDir, 'pydocs') 1060 curDir = os.getcwd() 1061 os.chdir(buildDir) 1062 runCommand('make clean') 1063 1064 # Search third-party source directory for a pre-built version of the docs. 1065 # Use the naming convention of the docs.python.org html downloads: 1066 # python-3.9.0b1-docs-html.tar.bz2 1067 doctarfiles = [ f for f in os.listdir(DEPSRC) 1068 if f.startswith('python-'+getFullVersion()) 1069 if f.endswith('-docs-html.tar.bz2') ] 1070 if doctarfiles: 1071 doctarfile = doctarfiles[0] 1072 if not os.path.exists('build'): 1073 os.mkdir('build') 1074 # if build directory existed, it was emptied by make clean, above 1075 os.chdir('build') 1076 # Extract the first archive found for this version into build 1077 runCommand('tar xjf %s'%shellQuote(os.path.join(DEPSRC, doctarfile))) 1078 # see if tar extracted a directory ending in -docs-html 1079 archivefiles = [ f for f in os.listdir('.') 1080 if f.endswith('-docs-html') 1081 if os.path.isdir(f) ] 1082 if archivefiles: 1083 archivefile = archivefiles[0] 1084 # make it our 'Docs/build/html' directory 1085 print(' -- using pre-built python documentation from %s'%archivefile) 1086 os.rename(archivefile, 'html') 1087 os.chdir(buildDir) 1088 1089 htmlDir = os.path.join('build', 'html') 1090 if not os.path.exists(htmlDir): 1091 # Create virtual environment for docs builds with blurb and sphinx 1092 runCommand('make venv') 1093 runCommand('venv/bin/python3 -m pip install -U Sphinx==2.3.1') 1094 runCommand('make html PYTHON=venv/bin/python') 1095 os.rename(htmlDir, docdir) 1096 os.chdir(curDir) 1097 1098 1099def buildPython(): 1100 print("Building a universal python for %s architectures" % UNIVERSALARCHS) 1101 1102 buildDir = os.path.join(WORKDIR, '_bld', 'python') 1103 rootDir = os.path.join(WORKDIR, '_root') 1104 1105 if os.path.exists(buildDir): 1106 shutil.rmtree(buildDir) 1107 if os.path.exists(rootDir): 1108 shutil.rmtree(rootDir) 1109 os.makedirs(buildDir) 1110 os.makedirs(rootDir) 1111 os.makedirs(os.path.join(rootDir, 'empty-dir')) 1112 curdir = os.getcwd() 1113 os.chdir(buildDir) 1114 1115 # Extract the version from the configure file, needed to calculate 1116 # several paths. 1117 version = getVersion() 1118 1119 # Since the extra libs are not in their installed framework location 1120 # during the build, augment the library path so that the interpreter 1121 # will find them during its extension import sanity checks. 1122 1123 print("Running configure...") 1124 runCommand("%s -C --enable-framework --enable-universalsdk=/ " 1125 "--with-universal-archs=%s " 1126 "%s " 1127 "%s " 1128 "%s " 1129 "%s " 1130 "%s " 1131 "LDFLAGS='-g -L%s/libraries/usr/local/lib' " 1132 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( 1133 shellQuote(os.path.join(SRCDIR, 'configure')), 1134 UNIVERSALARCHS, 1135 (' ', '--with-computed-gotos ')[PYTHON_3], 1136 (' ', '--without-ensurepip ')[PYTHON_3], 1137 (' ', "--with-openssl='%s/libraries/usr/local'"%( 1138 shellQuote(WORKDIR)[1:-1],))[PYTHON_3], 1139 (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( 1140 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1141 (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( 1142 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1143 shellQuote(WORKDIR)[1:-1], 1144 shellQuote(WORKDIR)[1:-1])) 1145 1146 # As of macOS 10.11 with SYSTEM INTEGRITY PROTECTION, DYLD_* 1147 # environment variables are no longer automatically inherited 1148 # by child processes from their parents. We used to just set 1149 # DYLD_LIBRARY_PATH, pointing to the third-party libs, 1150 # in build-installer.py's process environment and it was 1151 # passed through the make utility into the environment of 1152 # setup.py. Instead, we now append DYLD_LIBRARY_PATH to 1153 # the existing RUNSHARED configuration value when we call 1154 # make for extension module builds. 1155 1156 runshared_for_make = "".join([ 1157 " RUNSHARED=", 1158 "'", 1159 grepValue("Makefile", "RUNSHARED"), 1160 ' DYLD_LIBRARY_PATH=', 1161 os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib'), 1162 "'" ]) 1163 1164 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS 1165 # and, if defined, append its value to the make command. This allows 1166 # us to pass in version control tags, like GITTAG, to a build from a 1167 # tarball rather than from a vcs checkout, thus eliminating the need 1168 # to have a working copy of the vcs program on the build machine. 1169 # 1170 # A typical use might be: 1171 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ 1172 # GITVERSION='echo 123456789a' \ 1173 # GITTAG='echo v3.6.0' \ 1174 # GITBRANCH='echo 3.6'" 1175 1176 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") 1177 if make_extras: 1178 make_cmd = "make " + make_extras + runshared_for_make 1179 else: 1180 make_cmd = "make" + runshared_for_make 1181 print("Running " + make_cmd) 1182 runCommand(make_cmd) 1183 1184 make_cmd = "make install DESTDIR=%s %s"%( 1185 shellQuote(rootDir), 1186 runshared_for_make) 1187 print("Running " + make_cmd) 1188 runCommand(make_cmd) 1189 1190 make_cmd = "make frameworkinstallextras DESTDIR=%s %s"%( 1191 shellQuote(rootDir), 1192 runshared_for_make) 1193 print("Running " + make_cmd) 1194 runCommand(make_cmd) 1195 1196 print("Copying required shared libraries") 1197 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): 1198 build_lib_dir = os.path.join( 1199 WORKDIR, 'libraries', 'Library', 'Frameworks', 1200 'Python.framework', 'Versions', getVersion(), 'lib') 1201 fw_lib_dir = os.path.join( 1202 WORKDIR, '_root', 'Library', 'Frameworks', 1203 'Python.framework', 'Versions', getVersion(), 'lib') 1204 if internalTk(): 1205 # move Tcl and Tk pkgconfig files 1206 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( 1207 shellQuote(build_lib_dir), 1208 shellQuote(fw_lib_dir) )) 1209 runCommand("rm -r %s/pkgconfig"%( 1210 shellQuote(build_lib_dir), )) 1211 runCommand("mv %s/* %s"%( 1212 shellQuote(build_lib_dir), 1213 shellQuote(fw_lib_dir) )) 1214 1215 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') 1216 frmDirVersioned = os.path.join(frmDir, 'Versions', version) 1217 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,)) 1218 # create directory for OpenSSL certificates 1219 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl') 1220 os.makedirs(sslDir) 1221 1222 print("Fix file modes") 1223 gid = grp.getgrnam('admin').gr_gid 1224 1225 shared_lib_error = False 1226 for dirpath, dirnames, filenames in os.walk(frmDir): 1227 for dn in dirnames: 1228 os.chmod(os.path.join(dirpath, dn), STAT_0o775) 1229 os.chown(os.path.join(dirpath, dn), -1, gid) 1230 1231 for fn in filenames: 1232 if os.path.islink(fn): 1233 continue 1234 1235 # "chmod g+w $fn" 1236 p = os.path.join(dirpath, fn) 1237 st = os.stat(p) 1238 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP) 1239 os.chown(p, -1, gid) 1240 1241 if fn in EXPECTED_SHARED_LIBS: 1242 # check to see that this file was linked with the 1243 # expected library path and version 1244 data = captureCommand("otool -L %s" % shellQuote(p)) 1245 for sl in EXPECTED_SHARED_LIBS[fn]: 1246 if ("\t%s " % sl) not in data: 1247 print("Expected shared lib %s was not linked with %s" 1248 % (sl, p)) 1249 shared_lib_error = True 1250 1251 if shared_lib_error: 1252 fatal("Unexpected shared library errors.") 1253 1254 if PYTHON_3: 1255 LDVERSION=None 1256 VERSION=None 1257 ABIFLAGS=None 1258 1259 fp = open(os.path.join(buildDir, 'Makefile'), 'r') 1260 for ln in fp: 1261 if ln.startswith('VERSION='): 1262 VERSION=ln.split()[1] 1263 if ln.startswith('ABIFLAGS='): 1264 ABIFLAGS=ln.split() 1265 ABIFLAGS=ABIFLAGS[1] if len(ABIFLAGS) > 1 else '' 1266 if ln.startswith('LDVERSION='): 1267 LDVERSION=ln.split()[1] 1268 fp.close() 1269 1270 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION) 1271 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS) 1272 config_suffix = '-' + LDVERSION 1273 if getVersionMajorMinor() >= (3, 6): 1274 config_suffix = config_suffix + '-darwin' 1275 else: 1276 config_suffix = '' # Python 2.x 1277 1278 # We added some directories to the search path during the configure 1279 # phase. Remove those because those directories won't be there on 1280 # the end-users system. Also remove the directories from _sysconfigdata.py 1281 # (added in 3.3) if it exists. 1282 1283 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,) 1284 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,) 1285 1286 # fix Makefile 1287 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile') 1288 fp = open(path, 'r') 1289 data = fp.read() 1290 fp.close() 1291 1292 for p in (include_path, lib_path): 1293 data = data.replace(" " + p, '') 1294 data = data.replace(p + " ", '') 1295 1296 fp = open(path, 'w') 1297 fp.write(data) 1298 fp.close() 1299 1300 # fix _sysconfigdata 1301 # 1302 # TODO: make this more robust! test_sysconfig_module of 1303 # distutils.tests.test_sysconfig.SysconfigTestCase tests that 1304 # the output from get_config_var in both sysconfig and 1305 # distutils.sysconfig is exactly the same for both CFLAGS and 1306 # LDFLAGS. The fixing up is now complicated by the pretty 1307 # printing in _sysconfigdata.py. Also, we are using the 1308 # pprint from the Python running the installer build which 1309 # may not cosmetically format the same as the pprint in the Python 1310 # being built (and which is used to originally generate 1311 # _sysconfigdata.py). 1312 1313 import pprint 1314 if getVersionMajorMinor() >= (3, 6): 1315 # XXX this is extra-fragile 1316 path = os.path.join(path_to_lib, 1317 '_sysconfigdata_%s_darwin_darwin.py' % (ABIFLAGS,)) 1318 else: 1319 path = os.path.join(path_to_lib, '_sysconfigdata.py') 1320 fp = open(path, 'r') 1321 data = fp.read() 1322 fp.close() 1323 # create build_time_vars dict 1324 if RUNNING_ON_PYTHON2: 1325 exec(data) 1326 else: 1327 g_dict = {} 1328 l_dict = {} 1329 exec(data, g_dict, l_dict) 1330 build_time_vars = l_dict['build_time_vars'] 1331 vars = {} 1332 for k, v in build_time_vars.items(): 1333 if type(v) == type(''): 1334 for p in (include_path, lib_path): 1335 v = v.replace(' ' + p, '') 1336 v = v.replace(p + ' ', '') 1337 vars[k] = v 1338 1339 fp = open(path, 'w') 1340 # duplicated from sysconfig._generate_posix_vars() 1341 fp.write('# system configuration generated and used by' 1342 ' the sysconfig module\n') 1343 fp.write('build_time_vars = ') 1344 pprint.pprint(vars, stream=fp) 1345 fp.close() 1346 1347 # Add symlinks in /usr/local/bin, using relative links 1348 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin') 1349 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks', 1350 'Python.framework', 'Versions', version, 'bin') 1351 if os.path.exists(usr_local_bin): 1352 shutil.rmtree(usr_local_bin) 1353 os.makedirs(usr_local_bin) 1354 for fn in os.listdir( 1355 os.path.join(frmDir, 'Versions', version, 'bin')): 1356 os.symlink(os.path.join(to_framework, fn), 1357 os.path.join(usr_local_bin, fn)) 1358 1359 os.chdir(curdir) 1360 1361def patchFile(inPath, outPath): 1362 data = fileContents(inPath) 1363 data = data.replace('$FULL_VERSION', getFullVersion()) 1364 data = data.replace('$VERSION', getVersion()) 1365 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later'))) 1366 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS])) 1367 data = data.replace('$INSTALL_SIZE', installSize()) 1368 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS)) 1369 1370 # This one is not handy as a template variable 1371 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework') 1372 fp = open(outPath, 'w') 1373 fp.write(data) 1374 fp.close() 1375 1376def patchScript(inPath, outPath): 1377 major, minor = getVersionMajorMinor() 1378 data = fileContents(inPath) 1379 data = data.replace('@PYMAJOR@', str(major)) 1380 data = data.replace('@PYVER@', getVersion()) 1381 fp = open(outPath, 'w') 1382 fp.write(data) 1383 fp.close() 1384 os.chmod(outPath, STAT_0o755) 1385 1386 1387 1388def packageFromRecipe(targetDir, recipe): 1389 curdir = os.getcwd() 1390 try: 1391 # The major version (such as 2.5) is included in the package name 1392 # because having two version of python installed at the same time is 1393 # common. 1394 pkgname = '%s-%s'%(recipe['name'], getVersion()) 1395 srcdir = recipe.get('source') 1396 pkgroot = recipe.get('topdir', srcdir) 1397 postflight = recipe.get('postflight') 1398 readme = textwrap.dedent(recipe['readme']) 1399 isRequired = recipe.get('required', True) 1400 1401 print("- building package %s"%(pkgname,)) 1402 1403 # Substitute some variables 1404 textvars = dict( 1405 VER=getVersion(), 1406 FULLVER=getFullVersion(), 1407 ) 1408 readme = readme % textvars 1409 1410 if pkgroot is not None: 1411 pkgroot = pkgroot % textvars 1412 else: 1413 pkgroot = '/' 1414 1415 if srcdir is not None: 1416 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:]) 1417 srcdir = srcdir % textvars 1418 1419 if postflight is not None: 1420 postflight = os.path.abspath(postflight) 1421 1422 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents') 1423 os.makedirs(packageContents) 1424 1425 if srcdir is not None: 1426 os.chdir(srcdir) 1427 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1428 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1429 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),)) 1430 1431 fn = os.path.join(packageContents, 'PkgInfo') 1432 fp = open(fn, 'w') 1433 fp.write('pmkrpkg1') 1434 fp.close() 1435 1436 rsrcDir = os.path.join(packageContents, "Resources") 1437 os.mkdir(rsrcDir) 1438 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w') 1439 fp.write(readme) 1440 fp.close() 1441 1442 if postflight is not None: 1443 patchScript(postflight, os.path.join(rsrcDir, 'postflight')) 1444 1445 vers = getFullVersion() 1446 major, minor = getVersionMajorMinor() 1447 pl = dict( 1448 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), 1449 CFBundleIdentifier='org.python.Python.%s'%(pkgname,), 1450 CFBundleName='Python.%s'%(pkgname,), 1451 CFBundleShortVersionString=vers, 1452 IFMajorVersion=major, 1453 IFMinorVersion=minor, 1454 IFPkgFormatVersion=0.10000000149011612, 1455 IFPkgFlagAllowBackRev=False, 1456 IFPkgFlagAuthorizationAction="RootAuthorization", 1457 IFPkgFlagDefaultLocation=pkgroot, 1458 IFPkgFlagFollowLinks=True, 1459 IFPkgFlagInstallFat=True, 1460 IFPkgFlagIsRequired=isRequired, 1461 IFPkgFlagOverwritePermissions=False, 1462 IFPkgFlagRelocatable=False, 1463 IFPkgFlagRestartAction="NoRestart", 1464 IFPkgFlagRootVolumeOnly=True, 1465 IFPkgFlagUpdateInstalledLangauges=False, 1466 ) 1467 writePlist(pl, os.path.join(packageContents, 'Info.plist')) 1468 1469 pl = dict( 1470 IFPkgDescriptionDescription=readme, 1471 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), 1472 IFPkgDescriptionVersion=vers, 1473 ) 1474 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist')) 1475 1476 finally: 1477 os.chdir(curdir) 1478 1479 1480def makeMpkgPlist(path): 1481 1482 vers = getFullVersion() 1483 major, minor = getVersionMajorMinor() 1484 1485 pl = dict( 1486 CFBundleGetInfoString="Python %s"%(vers,), 1487 CFBundleIdentifier='org.python.Python', 1488 CFBundleName='Python', 1489 CFBundleShortVersionString=vers, 1490 IFMajorVersion=major, 1491 IFMinorVersion=minor, 1492 IFPkgFlagComponentDirectory="Contents/Packages", 1493 IFPkgFlagPackageList=[ 1494 dict( 1495 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()), 1496 IFPkgFlagPackageSelection=item.get('selected', 'selected'), 1497 ) 1498 for item in pkg_recipes() 1499 ], 1500 IFPkgFormatVersion=0.10000000149011612, 1501 IFPkgFlagBackgroundScaling="proportional", 1502 IFPkgFlagBackgroundAlignment="left", 1503 IFPkgFlagAuthorizationAction="RootAuthorization", 1504 ) 1505 1506 writePlist(pl, path) 1507 1508 1509def buildInstaller(): 1510 1511 # Zap all compiled files 1512 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')): 1513 for fn in filenames: 1514 if fn.endswith('.pyc') or fn.endswith('.pyo'): 1515 os.unlink(os.path.join(dirpath, fn)) 1516 1517 outdir = os.path.join(WORKDIR, 'installer') 1518 if os.path.exists(outdir): 1519 shutil.rmtree(outdir) 1520 os.mkdir(outdir) 1521 1522 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents') 1523 pkgcontents = os.path.join(pkgroot, 'Packages') 1524 os.makedirs(pkgcontents) 1525 for recipe in pkg_recipes(): 1526 packageFromRecipe(pkgcontents, recipe) 1527 1528 rsrcDir = os.path.join(pkgroot, 'Resources') 1529 1530 fn = os.path.join(pkgroot, 'PkgInfo') 1531 fp = open(fn, 'w') 1532 fp.write('pmkrpkg1') 1533 fp.close() 1534 1535 os.mkdir(rsrcDir) 1536 1537 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) 1538 pl = dict( 1539 IFPkgDescriptionTitle="Python", 1540 IFPkgDescriptionVersion=getVersion(), 1541 ) 1542 1543 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist')) 1544 for fn in os.listdir('resources'): 1545 if fn == '.svn': continue 1546 if fn.endswith('.jpg'): 1547 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1548 else: 1549 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1550 1551 1552def installSize(clear=False, _saved=[]): 1553 if clear: 1554 del _saved[:] 1555 if not _saved: 1556 data = captureCommand("du -ks %s"%( 1557 shellQuote(os.path.join(WORKDIR, '_root')))) 1558 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),)) 1559 return _saved[0] 1560 1561 1562def buildDMG(): 1563 """ 1564 Create DMG containing the rootDir. 1565 """ 1566 outdir = os.path.join(WORKDIR, 'diskimage') 1567 if os.path.exists(outdir): 1568 shutil.rmtree(outdir) 1569 1570 imagepath = os.path.join(outdir, 1571 'python-%s-macosx%s'%(getFullVersion(),DEPTARGET)) 1572 if INCLUDE_TIMESTAMP: 1573 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3]) 1574 imagepath = imagepath + '.dmg' 1575 1576 os.mkdir(outdir) 1577 1578 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9, 1579 # when hdiutil create fails with "Resource busy". For now, just retry 1580 # the create a few times and hope that it eventually works. 1581 1582 volname='Python %s'%(getFullVersion()) 1583 cmd = ("hdiutil create -format UDRW -volname %s -srcfolder %s -size 100m %s"%( 1584 shellQuote(volname), 1585 shellQuote(os.path.join(WORKDIR, 'installer')), 1586 shellQuote(imagepath + ".tmp.dmg" ))) 1587 for i in range(5): 1588 fd = os.popen(cmd, 'r') 1589 data = fd.read() 1590 xit = fd.close() 1591 if not xit: 1592 break 1593 sys.stdout.write(data) 1594 print(" -- retrying hdiutil create") 1595 time.sleep(5) 1596 else: 1597 raise RuntimeError("command failed: %s"%(cmd,)) 1598 1599 if not os.path.exists(os.path.join(WORKDIR, "mnt")): 1600 os.mkdir(os.path.join(WORKDIR, "mnt")) 1601 runCommand("hdiutil attach %s -mountroot %s"%( 1602 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt")))) 1603 1604 # Custom icon for the DMG, shown when the DMG is mounted. 1605 shutil.copy("../Icons/Disk Image.icns", 1606 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns")) 1607 runCommand("SetFile -a C %s/"%( 1608 shellQuote(os.path.join(WORKDIR, "mnt", volname)),)) 1609 1610 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname)))) 1611 1612 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns") 1613 runCommand("hdiutil convert %s -format UDZO -o %s"%( 1614 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath))) 1615 setIcon(imagepath, "../Icons/Disk Image.icns") 1616 1617 os.unlink(imagepath + ".tmp.dmg") 1618 1619 return imagepath 1620 1621 1622def setIcon(filePath, icnsPath): 1623 """ 1624 Set the custom icon for the specified file or directory. 1625 """ 1626 1627 dirPath = os.path.normpath(os.path.dirname(__file__)) 1628 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon") 1629 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime: 1630 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due 1631 # to connections to the window server. 1632 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS") 1633 if not os.path.exists(appPath): 1634 os.makedirs(appPath) 1635 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%( 1636 shellQuote(toolPath), shellQuote(dirPath))) 1637 1638 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath), 1639 shellQuote(filePath))) 1640 1641def main(): 1642 # First parse options and check if we can perform our work 1643 parseOptions() 1644 checkEnvironment() 1645 1646 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET 1647 os.environ['CC'] = CC 1648 os.environ['CXX'] = CXX 1649 1650 if os.path.exists(WORKDIR): 1651 shutil.rmtree(WORKDIR) 1652 os.mkdir(WORKDIR) 1653 1654 os.environ['LC_ALL'] = 'C' 1655 1656 # Then build third-party libraries such as sleepycat DB4. 1657 buildLibraries() 1658 1659 # Now build python itself 1660 buildPython() 1661 1662 # And then build the documentation 1663 # Remove the Deployment Target from the shell 1664 # environment, it's no longer needed and 1665 # an unexpected build target can cause problems 1666 # when Sphinx and its dependencies need to 1667 # be (re-)installed. 1668 del os.environ['MACOSX_DEPLOYMENT_TARGET'] 1669 buildPythonDocs() 1670 1671 1672 # Prepare the applications folder 1673 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%( 1674 getVersion(),)) 1675 fn = os.path.join(folder, "License.rtf") 1676 patchFile("resources/License.rtf", fn) 1677 fn = os.path.join(folder, "ReadMe.rtf") 1678 patchFile("resources/ReadMe.rtf", fn) 1679 fn = os.path.join(folder, "Update Shell Profile.command") 1680 patchScript("scripts/postflight.patch-profile", fn) 1681 fn = os.path.join(folder, "Install Certificates.command") 1682 patchScript("resources/install_certificates.command", fn) 1683 os.chmod(folder, STAT_0o755) 1684 setIcon(folder, "../Icons/Python Folder.icns") 1685 1686 # Create the installer 1687 buildInstaller() 1688 1689 # And copy the readme into the directory containing the installer 1690 patchFile('resources/ReadMe.rtf', 1691 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf')) 1692 1693 # Ditto for the license file. 1694 patchFile('resources/License.rtf', 1695 os.path.join(WORKDIR, 'installer', 'License.rtf')) 1696 1697 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w') 1698 fp.write("# BUILD INFO\n") 1699 fp.write("# Date: %s\n" % time.ctime()) 1700 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos) 1701 fp.close() 1702 1703 # And copy it to a DMG 1704 buildDMG() 1705 1706if __name__ == "__main__": 1707 main() 1708