1#!/usr/bin/env python 2""" 3This script is used to build "official" universal installers on macOS. 4 5NEW for 3.10 and backports: 6- support universal2 variant with arm64 and x86_64 archs 7- enable clang optimizations when building on 10.15+ 8 9NEW for 3.9.0 and backports: 10- 2.7 end-of-life issues: 11 - Python 3 installs now update the Current version link 12 in /Library/Frameworks/Python.framework/Versions 13- fully support running under Python 3 as well as 2.7 14- support building on newer macOS systems with SIP 15- fully support building on macOS 10.9+ 16- support 10.6+ on best effort 17- support bypassing docs build by supplying a prebuilt 18 docs html tarball in the third-party source library, 19 in the format and filename conventional of those 20 downloadable from python.org: 21 python-3.x.y-docs-html.tar.bz2 22 23NEW for 3.7.0: 24- support Intel 64-bit-only () and 32-bit-only installer builds 25- build and use internal Tcl/Tk 8.6 for 10.6+ builds 26- deprecate use of explicit SDK (--sdk-path=) since all but the oldest 27 versions of Xcode support implicit setting of an SDK via environment 28 variables (SDKROOT and friends, see the xcrun man page for more info). 29 The SDK stuff was primarily needed for building universal installers 30 for 10.4; so as of 3.7.0, building installers for 10.4 is no longer 31 supported with build-installer. 32- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" 33 34TODO: 35- test building with SDKROOT and DEVELOPER_DIR xcrun env variables 36 37Usage: see USAGE variable in the script. 38""" 39import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp 40try: 41 import urllib2 as urllib_request 42except ImportError: 43 import urllib.request as urllib_request 44 45STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 46 | stat.S_IRGRP | stat.S_IXGRP 47 | stat.S_IROTH | stat.S_IXOTH ) 48 49STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 50 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP 51 | stat.S_IROTH | stat.S_IXOTH ) 52 53INCLUDE_TIMESTAMP = 1 54VERBOSE = 1 55 56RUNNING_ON_PYTHON2 = sys.version_info.major == 2 57 58if RUNNING_ON_PYTHON2: 59 from plistlib import writePlist 60else: 61 from plistlib import dump 62 def writePlist(path, plist): 63 with open(plist, 'wb') as fp: 64 dump(path, fp) 65 66def shellQuote(value): 67 """ 68 Return the string value in a form that can safely be inserted into 69 a shell command. 70 """ 71 return "'%s'"%(value.replace("'", "'\"'\"'")) 72 73def grepValue(fn, variable): 74 """ 75 Return the unquoted value of a variable from a file.. 76 QUOTED_VALUE='quotes' -> str('quotes') 77 UNQUOTED_VALUE=noquotes -> str('noquotes') 78 """ 79 variable = variable + '=' 80 for ln in open(fn, 'r'): 81 if ln.startswith(variable): 82 value = ln[len(variable):].strip() 83 return value.strip("\"'") 84 raise RuntimeError("Cannot find variable %s" % variable[:-1]) 85 86_cache_getVersion = None 87 88def getVersion(): 89 global _cache_getVersion 90 if _cache_getVersion is None: 91 _cache_getVersion = grepValue( 92 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION') 93 return _cache_getVersion 94 95def getVersionMajorMinor(): 96 return tuple([int(n) for n in getVersion().split('.', 2)]) 97 98_cache_getFullVersion = None 99 100def getFullVersion(): 101 global _cache_getFullVersion 102 if _cache_getFullVersion is not None: 103 return _cache_getFullVersion 104 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h') 105 for ln in open(fn): 106 if 'PY_VERSION' in ln: 107 _cache_getFullVersion = ln.split()[-1][1:-1] 108 return _cache_getFullVersion 109 raise RuntimeError("Cannot find full version??") 110 111FW_PREFIX = ["Library", "Frameworks", "Python.framework"] 112FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions 113FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions 114 115# The directory we'll use to create the build (will be erased and recreated) 116WORKDIR = "/tmp/_py" 117 118# The directory we'll use to store third-party sources. Set this to something 119# else if you don't want to re-fetch required libraries every time. 120DEPSRC = os.path.join(WORKDIR, 'third-party') 121DEPSRC = os.path.expanduser('~/Universal/other-sources') 122 123universal_opts_map = { 'universal2': ('arm64', 'x86_64'), 124 '32-bit': ('i386', 'ppc',), 125 '64-bit': ('x86_64', 'ppc64',), 126 'intel': ('i386', 'x86_64'), 127 'intel-32': ('i386',), 128 'intel-64': ('x86_64',), 129 '3-way': ('ppc', 'i386', 'x86_64'), 130 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } 131default_target_map = { 132 'universal2': '10.9', 133 '64-bit': '10.5', 134 '3-way': '10.5', 135 'intel': '10.5', 136 'intel-32': '10.4', 137 'intel-64': '10.5', 138 'all': '10.5', 139} 140 141UNIVERSALOPTS = tuple(universal_opts_map.keys()) 142 143UNIVERSALARCHS = '32-bit' 144 145ARCHLIST = universal_opts_map[UNIVERSALARCHS] 146 147# Source directory (assume we're in Mac/BuildScript) 148SRCDIR = os.path.dirname( 149 os.path.dirname( 150 os.path.dirname( 151 os.path.abspath(__file__ 152 )))) 153 154# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level 155DEPTARGET = '10.5' 156 157def getDeptargetTuple(): 158 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) 159 160def getBuildTuple(): 161 return tuple([int(n) for n in platform.mac_ver()[0].split('.')[0:2]]) 162 163def getTargetCompilers(): 164 target_cc_map = { 165 '10.4': ('gcc-4.0', 'g++-4.0'), 166 '10.5': ('gcc', 'g++'), 167 '10.6': ('gcc', 'g++'), 168 '10.7': ('gcc', 'g++'), 169 '10.8': ('gcc', 'g++'), 170 } 171 return target_cc_map.get(DEPTARGET, ('clang', 'clang++') ) 172 173CC, CXX = getTargetCompilers() 174 175PYTHON_3 = getVersionMajorMinor() >= (3, 0) 176 177USAGE = textwrap.dedent("""\ 178 Usage: build_python [options] 179 180 Options: 181 -? or -h: Show this message 182 -b DIR 183 --build-dir=DIR: Create build here (default: %(WORKDIR)r) 184 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) 185 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) 186 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) 187 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) 188 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) 189""")% globals() 190 191# Dict of object file names with shared library names to check after building. 192# This is to ensure that we ended up dynamically linking with the shared 193# library paths and versions we expected. For example: 194# EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 195# '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl', 196# '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] 197EXPECTED_SHARED_LIBS = {} 198 199# Are we building and linking with our own copy of Tcl/TK? 200# For now, do so if deployment target is 10.6+. 201def internalTk(): 202 return getDeptargetTuple() >= (10, 6) 203 204# Do we use 8.6.8 when building our own copy 205# of Tcl/Tk or a modern version. 206# We use the old version when buildin on 207# old versions of macOS due to build issues. 208def useOldTk(): 209 return getBuildTuple() < (10, 15) 210 211 212def tweak_tcl_build(basedir, archList): 213 with open("Makefile", "r") as fp: 214 contents = fp.readlines() 215 216 # For reasons I don't understand the tcl configure script 217 # decides that some stdlib symbols aren't present, before 218 # deciding that strtod is broken. 219 new_contents = [] 220 for line in contents: 221 if line.startswith("COMPAT_OBJS"): 222 # note: the space before strtod.o is intentional, 223 # the detection of a broken strtod results in 224 # "fixstrod.o" on this line. 225 for nm in ("strstr.o", "strtoul.o", " strtod.o"): 226 line = line.replace(nm, "") 227 new_contents.append(line) 228 229 with open("Makefile", "w") as fp: 230 fp.writelines(new_contents) 231 232# List of names of third party software built with this installer. 233# The names will be inserted into the rtf version of the License. 234THIRD_PARTY_LIBS = [] 235 236# Instructions for building libraries that are necessary for building a 237# batteries included python. 238# [The recipes are defined here for convenience but instantiated later after 239# command line options have been processed.] 240def library_recipes(): 241 result = [] 242 243 # Since Apple removed the header files for the deprecated system 244 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not 245 # have much choice but to build our own copy here, too. 246 247 result.extend([ 248 dict( 249 name="OpenSSL 1.1.1m", 250 url="https://www.openssl.org/source/openssl-1.1.1m.tar.gz", 251 checksum='8ec70f665c145c3103f6e330f538a9db', 252 buildrecipe=build_universal_openssl, 253 configure=None, 254 install=None, 255 ), 256 ]) 257 258 if internalTk(): 259 if useOldTk(): 260 tcl_tk_ver='8.6.8' 261 tcl_checksum='81656d3367af032e0ae6157eff134f89' 262 263 tk_checksum='5e0faecba458ee1386078fb228d008ba' 264 tk_patches = ['tk868_on_10_8_10_9.patch'] 265 266 else: 267 tcl_tk_ver='8.6.12' 268 tcl_checksum='87ea890821d2221f2ab5157bc5eb885f' 269 270 tk_checksum='1d6dcf6120356e3d211e056dff5e462a' 271 tk_patches = [ ] 272 273 274 result.extend([ 275 dict( 276 name="Tcl %s"%(tcl_tk_ver,), 277 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl%s-src.tar.gz"%(tcl_tk_ver,), 278 checksum=tcl_checksum, 279 buildDir="unix", 280 configure_pre=[ 281 '--enable-shared', 282 '--enable-threads', 283 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 284 ], 285 useLDFlags=False, 286 buildrecipe=tweak_tcl_build, 287 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 288 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 289 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 290 }, 291 ), 292 dict( 293 name="Tk %s"%(tcl_tk_ver,), 294 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk%s-src.tar.gz"%(tcl_tk_ver,), 295 checksum=tk_checksum, 296 patches=tk_patches, 297 buildDir="unix", 298 configure_pre=[ 299 '--enable-aqua', 300 '--enable-shared', 301 '--enable-threads', 302 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 303 ], 304 useLDFlags=False, 305 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 306 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 307 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 308 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), 309 }, 310 ), 311 ]) 312 313 if PYTHON_3: 314 result.extend([ 315 dict( 316 name="XZ 5.2.3", 317 url="http://tukaani.org/xz/xz-5.2.3.tar.gz", 318 checksum='ef68674fb47a8b8e741b34e429d86e9d', 319 configure_pre=[ 320 '--disable-dependency-tracking', 321 ] 322 ), 323 ]) 324 325 result.extend([ 326 dict( 327 name="NCurses 5.9", 328 url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz", 329 checksum='8cb9c412e5f2d96bc6f459aa8c6282a1', 330 configure_pre=[ 331 "--enable-widec", 332 "--without-cxx", 333 "--without-cxx-binding", 334 "--without-ada", 335 "--without-curses-h", 336 "--enable-shared", 337 "--with-shared", 338 "--without-debug", 339 "--without-normal", 340 "--without-tests", 341 "--without-manpages", 342 "--datadir=/usr/share", 343 "--sysconfdir=/etc", 344 "--sharedstatedir=/usr/com", 345 "--with-terminfo-dirs=/usr/share/terminfo", 346 "--with-default-terminfo-dir=/usr/share/terminfo", 347 "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), 348 ], 349 patchscripts=[ 350 ("ftp://ftp.invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", 351 "f54bf02a349f96a7c4f0d00922f3a0d4"), 352 ], 353 useLDFlags=False, 354 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%( 355 shellQuote(os.path.join(WORKDIR, 'libraries')), 356 shellQuote(os.path.join(WORKDIR, 'libraries')), 357 getVersion(), 358 ), 359 ), 360 dict( 361 name="SQLite 3.35.5", 362 url="https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz", 363 checksum='d1d1aba394c8e0443077dc9f1a681bb8', 364 extra_cflags=('-Os ' 365 '-DSQLITE_ENABLE_FTS5 ' 366 '-DSQLITE_ENABLE_FTS4 ' 367 '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' 368 '-DSQLITE_ENABLE_JSON1 ' 369 '-DSQLITE_ENABLE_RTREE ' 370 '-DSQLITE_OMIT_AUTOINIT ' 371 '-DSQLITE_TCL=0 ' 372 ), 373 configure_pre=[ 374 '--enable-threadsafe', 375 '--enable-shared=no', 376 '--enable-static=yes', 377 '--disable-readline', 378 '--disable-dependency-tracking', 379 ] 380 ), 381 ]) 382 383 if not PYTHON_3: 384 result.extend([ 385 dict( 386 name="Sleepycat DB 4.7.25", 387 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz", 388 checksum='ec2b87e833779681a0c3a814aa71359e', 389 buildDir="build_unix", 390 configure="../dist/configure", 391 configure_pre=[ 392 '--includedir=/usr/local/include/db4', 393 ] 394 ), 395 ]) 396 397 return result 398 399def compilerCanOptimize(): 400 """ 401 Return True iff the default Xcode version can use PGO and LTO 402 """ 403 # bpo-42235: The version check is pretty conservative, can be 404 # adjusted after testing 405 mac_ver = tuple(map(int, platform.mac_ver()[0].split('.'))) 406 return mac_ver >= (10, 15) 407 408# Instructions for building packages inside the .mpkg. 409def pkg_recipes(): 410 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3] 411 result = [ 412 dict( 413 name="PythonFramework", 414 long_name="Python Framework", 415 source="/Library/Frameworks/Python.framework", 416 readme="""\ 417 This package installs Python.framework, that is the python 418 interpreter and the standard library. 419 """, 420 postflight="scripts/postflight.framework", 421 selected='selected', 422 ), 423 dict( 424 name="PythonApplications", 425 long_name="GUI Applications", 426 source="/Applications/Python %(VER)s", 427 readme="""\ 428 This package installs IDLE (an interactive Python IDE), 429 Python Launcher and Build Applet (create application bundles 430 from python scripts). 431 432 It also installs a number of examples and demos. 433 """, 434 required=False, 435 selected='selected', 436 ), 437 dict( 438 name="PythonUnixTools", 439 long_name="UNIX command-line tools", 440 source="/usr/local/bin", 441 readme="""\ 442 This package installs the unix tools in /usr/local/bin for 443 compatibility with older releases of Python. This package 444 is not necessary to use Python. 445 """, 446 required=False, 447 selected='selected', 448 ), 449 dict( 450 name="PythonDocumentation", 451 long_name="Python Documentation", 452 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation", 453 source="/pydocs", 454 readme="""\ 455 This package installs the python documentation at a location 456 that is usable for pydoc and IDLE. 457 """, 458 postflight="scripts/postflight.documentation", 459 required=False, 460 selected='selected', 461 ), 462 dict( 463 name="PythonProfileChanges", 464 long_name="Shell profile updater", 465 readme="""\ 466 This packages updates your shell profile to make sure that 467 the Python tools are found by your shell in preference of 468 the system provided Python tools. 469 470 If you don't install this package you'll have to add 471 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin" 472 to your PATH by hand. 473 """, 474 postflight="scripts/postflight.patch-profile", 475 topdir="/Library/Frameworks/Python.framework", 476 source="/empty-dir", 477 required=False, 478 selected='selected', 479 ), 480 dict( 481 name="PythonInstallPip", 482 long_name="Install or upgrade pip", 483 readme="""\ 484 This package installs (or upgrades from an earlier version) 485 pip, a tool for installing and managing Python packages. 486 """, 487 postflight="scripts/postflight.ensurepip", 488 topdir="/Library/Frameworks/Python.framework", 489 source="/empty-dir", 490 required=False, 491 selected='selected', 492 ), 493 ] 494 495 return result 496 497def fatal(msg): 498 """ 499 A fatal error, bail out. 500 """ 501 sys.stderr.write('FATAL: ') 502 sys.stderr.write(msg) 503 sys.stderr.write('\n') 504 sys.exit(1) 505 506def fileContents(fn): 507 """ 508 Return the contents of the named file 509 """ 510 return open(fn, 'r').read() 511 512def runCommand(commandline): 513 """ 514 Run a command and raise RuntimeError if it fails. Output is suppressed 515 unless the command fails. 516 """ 517 fd = os.popen(commandline, 'r') 518 data = fd.read() 519 xit = fd.close() 520 if xit is not None: 521 sys.stdout.write(data) 522 raise RuntimeError("command failed: %s"%(commandline,)) 523 524 if VERBOSE: 525 sys.stdout.write(data); sys.stdout.flush() 526 527def captureCommand(commandline): 528 fd = os.popen(commandline, 'r') 529 data = fd.read() 530 xit = fd.close() 531 if xit is not None: 532 sys.stdout.write(data) 533 raise RuntimeError("command failed: %s"%(commandline,)) 534 535 return data 536 537def getTclTkVersion(configfile, versionline): 538 """ 539 search Tcl or Tk configuration file for version line 540 """ 541 try: 542 f = open(configfile, "r") 543 except OSError: 544 fatal("Framework configuration file not found: %s" % configfile) 545 546 for l in f: 547 if l.startswith(versionline): 548 f.close() 549 return l 550 551 fatal("Version variable %s not found in framework configuration file: %s" 552 % (versionline, configfile)) 553 554def checkEnvironment(): 555 """ 556 Check that we're running on a supported system. 557 """ 558 559 if sys.version_info[0:2] < (2, 7): 560 fatal("This script must be run with Python 2.7 (or later)") 561 562 if platform.system() != 'Darwin': 563 fatal("This script should be run on a macOS 10.5 (or later) system") 564 565 if int(platform.release().split('.')[0]) < 8: 566 fatal("This script should be run on a macOS 10.5 (or later) system") 567 568 # Because we only support dynamic load of only one major/minor version of 569 # Tcl/Tk, if we are not using building and using our own private copy of 570 # Tcl/Tk, ensure: 571 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked 572 # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer 573 # enforce that the version of the user-installed framework also 574 # exists in the system-supplied Tcl/Tk frameworks. Time to support 575 # Tcl/Tk 8.6 even if Apple does not. 576 if not internalTk(): 577 frameworks = {} 578 for framework in ['Tcl', 'Tk']: 579 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework 580 libfw = os.path.join('/', fwpth) 581 usrfw = os.path.join(os.getenv('HOME'), fwpth) 582 frameworks[framework] = os.readlink(libfw) 583 if not os.path.exists(libfw): 584 fatal("Please install a link to a current %s %s as %s so " 585 "the user can override the system framework." 586 % (framework, frameworks[framework], libfw)) 587 if os.path.exists(usrfw): 588 fatal("Please rename %s to avoid possible dynamic load issues." 589 % usrfw) 590 591 if frameworks['Tcl'] != frameworks['Tk']: 592 fatal("The Tcl and Tk frameworks are not the same version.") 593 594 print(" -- Building with external Tcl/Tk %s frameworks" 595 % frameworks['Tk']) 596 597 # add files to check after build 598 EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 599 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" 600 % frameworks['Tcl'], 601 "/Library/Frameworks/Tk.framework/Versions/%s/Tk" 602 % frameworks['Tk'], 603 ] 604 else: 605 print(" -- Building private copy of Tcl/Tk") 606 print("") 607 608 # Remove inherited environment variables which might influence build 609 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 610 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] 611 for ev in list(os.environ): 612 for prefix in environ_var_prefixes: 613 if ev.startswith(prefix) : 614 print("INFO: deleting environment variable %s=%s" % ( 615 ev, os.environ[ev])) 616 del os.environ[ev] 617 618 base_path = '/bin:/sbin:/usr/bin:/usr/sbin' 619 if 'SDK_TOOLS_BIN' in os.environ: 620 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path 621 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; 622 # add its fixed location here if it exists 623 OLD_DEVELOPER_TOOLS = '/Developer/Tools' 624 if os.path.isdir(OLD_DEVELOPER_TOOLS): 625 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS 626 os.environ['PATH'] = base_path 627 print("Setting default PATH: %s"%(os.environ['PATH'])) 628 629def parseOptions(args=None): 630 """ 631 Parse arguments and update global settings. 632 """ 633 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET 634 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX 635 global FW_VERSION_PREFIX 636 global FW_SSL_DIRECTORY 637 638 if args is None: 639 args = sys.argv[1:] 640 641 try: 642 options, args = getopt.getopt(args, '?hb', 643 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', 644 'dep-target=', 'universal-archs=', 'help' ]) 645 except getopt.GetoptError: 646 print(sys.exc_info()[1]) 647 sys.exit(1) 648 649 if args: 650 print("Additional arguments") 651 sys.exit(1) 652 653 deptarget = None 654 for k, v in options: 655 if k in ('-h', '-?', '--help'): 656 print(USAGE) 657 sys.exit(0) 658 659 elif k in ('-d', '--build-dir'): 660 WORKDIR=v 661 662 elif k in ('--third-party',): 663 DEPSRC=v 664 665 elif k in ('--sdk-path',): 666 print(" WARNING: --sdk-path is no longer supported") 667 668 elif k in ('--src-dir',): 669 SRCDIR=v 670 671 elif k in ('--dep-target', ): 672 DEPTARGET=v 673 deptarget=v 674 675 elif k in ('--universal-archs', ): 676 if v in UNIVERSALOPTS: 677 UNIVERSALARCHS = v 678 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 679 if deptarget is None: 680 # Select alternate default deployment 681 # target 682 DEPTARGET = default_target_map.get(v, '10.5') 683 else: 684 raise NotImplementedError(v) 685 686 else: 687 raise NotImplementedError(k) 688 689 SRCDIR=os.path.abspath(SRCDIR) 690 WORKDIR=os.path.abspath(WORKDIR) 691 DEPSRC=os.path.abspath(DEPSRC) 692 693 CC, CXX = getTargetCompilers() 694 695 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()] 696 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"] 697 698 print("-- Settings:") 699 print(" * Source directory: %s" % SRCDIR) 700 print(" * Build directory: %s" % WORKDIR) 701 print(" * Third-party source: %s" % DEPSRC) 702 print(" * Deployment target: %s" % DEPTARGET) 703 print(" * Universal archs: %s" % str(ARCHLIST)) 704 print(" * C compiler: %s" % CC) 705 print(" * C++ compiler: %s" % CXX) 706 print("") 707 print(" -- Building a Python %s framework at patch level %s" 708 % (getVersion(), getFullVersion())) 709 print("") 710 711def extractArchive(builddir, archiveName): 712 """ 713 Extract a source archive into 'builddir'. Returns the path of the 714 extracted archive. 715 716 XXX: This function assumes that archives contain a toplevel directory 717 that is has the same name as the basename of the archive. This is 718 safe enough for almost anything we use. Unfortunately, it does not 719 work for current Tcl and Tk source releases where the basename of 720 the archive ends with "-src" but the uncompressed directory does not. 721 For now, just special case Tcl and Tk tar.gz downloads. 722 """ 723 curdir = os.getcwd() 724 try: 725 os.chdir(builddir) 726 if archiveName.endswith('.tar.gz'): 727 retval = os.path.basename(archiveName[:-7]) 728 if ((retval.startswith('tcl') or retval.startswith('tk')) 729 and retval.endswith('-src')): 730 retval = retval[:-4] 731 if os.path.exists(retval): 732 shutil.rmtree(retval) 733 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') 734 735 elif archiveName.endswith('.tar.bz2'): 736 retval = os.path.basename(archiveName[:-8]) 737 if os.path.exists(retval): 738 shutil.rmtree(retval) 739 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r') 740 741 elif archiveName.endswith('.tar'): 742 retval = os.path.basename(archiveName[:-4]) 743 if os.path.exists(retval): 744 shutil.rmtree(retval) 745 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r') 746 747 elif archiveName.endswith('.zip'): 748 retval = os.path.basename(archiveName[:-4]) 749 if os.path.exists(retval): 750 shutil.rmtree(retval) 751 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r') 752 753 data = fp.read() 754 xit = fp.close() 755 if xit is not None: 756 sys.stdout.write(data) 757 raise RuntimeError("Cannot extract %s"%(archiveName,)) 758 759 return os.path.join(builddir, retval) 760 761 finally: 762 os.chdir(curdir) 763 764def downloadURL(url, fname): 765 """ 766 Download the contents of the url into the file. 767 """ 768 fpIn = urllib_request.urlopen(url) 769 fpOut = open(fname, 'wb') 770 block = fpIn.read(10240) 771 try: 772 while block: 773 fpOut.write(block) 774 block = fpIn.read(10240) 775 fpIn.close() 776 fpOut.close() 777 except: 778 try: 779 os.unlink(fname) 780 except OSError: 781 pass 782 783def verifyThirdPartyFile(url, checksum, fname): 784 """ 785 Download file from url to filename fname if it does not already exist. 786 Abort if file contents does not match supplied md5 checksum. 787 """ 788 name = os.path.basename(fname) 789 if os.path.exists(fname): 790 print("Using local copy of %s"%(name,)) 791 else: 792 print("Did not find local copy of %s"%(name,)) 793 print("Downloading %s"%(name,)) 794 downloadURL(url, fname) 795 print("Archive for %s stored as %s"%(name, fname)) 796 if os.system( 797 'MD5=$(openssl md5 %s) ; test "${MD5##*= }" = "%s"' 798 % (shellQuote(fname), checksum) ): 799 fatal('MD5 checksum mismatch for file %s' % fname) 800 801def build_universal_openssl(basedir, archList): 802 """ 803 Special case build recipe for universal build of openssl. 804 805 The upstream OpenSSL build system does not directly support 806 OS X universal builds. We need to build each architecture 807 separately then lipo them together into fat libraries. 808 """ 809 810 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4). 811 # If we are building on a 10.4.x or earlier system, 812 # unilaterally disable assembly code building to avoid the problem. 813 no_asm = int(platform.release().split(".")[0]) < 9 814 815 def build_openssl_arch(archbase, arch): 816 "Build one architecture of openssl" 817 arch_opts = { 818 "i386": ["darwin-i386-cc"], 819 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], 820 "arm64": ["darwin64-arm64-cc"], 821 "ppc": ["darwin-ppc-cc"], 822 "ppc64": ["darwin64-ppc-cc"], 823 } 824 825 # Somewhere between OpenSSL 1.1.0j and 1.1.1c, changes cause the 826 # "enable-ec_nistp_64_gcc_128" option to get compile errors when 827 # building on our 10.6 gcc-4.2 environment. There have been other 828 # reports of projects running into this when using older compilers. 829 # So, for now, do not try to use "enable-ec_nistp_64_gcc_128" when 830 # building for 10.6. 831 if getDeptargetTuple() == (10, 6): 832 arch_opts['x86_64'].remove('enable-ec_nistp_64_gcc_128') 833 834 configure_opts = [ 835 "no-idea", 836 "no-mdc2", 837 "no-rc5", 838 "no-zlib", 839 "no-ssl3", 840 # "enable-unit-test", 841 "shared", 842 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), 843 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), 844 ] 845 if no_asm: 846 configure_opts.append("no-asm") 847 runCommand(" ".join(["perl", "Configure"] 848 + arch_opts[arch] + configure_opts)) 849 runCommand("make depend") 850 runCommand("make all") 851 runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase)) 852 # runCommand("make test") 853 return 854 855 srcdir = os.getcwd() 856 universalbase = os.path.join(srcdir, "..", 857 os.path.basename(srcdir) + "-universal") 858 os.mkdir(universalbase) 859 archbasefws = [] 860 for arch in archList: 861 # fresh copy of the source tree 862 archsrc = os.path.join(universalbase, arch, "src") 863 shutil.copytree(srcdir, archsrc, symlinks=True) 864 # install base for this arch 865 archbase = os.path.join(universalbase, arch, "root") 866 os.mkdir(archbase) 867 # Python framework base within install_prefix: 868 # the build will install into this framework.. 869 # This is to ensure that the resulting shared libs have 870 # the desired real install paths built into them. 871 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX) 872 873 # build one architecture 874 os.chdir(archsrc) 875 build_openssl_arch(archbase, arch) 876 os.chdir(srcdir) 877 archbasefws.append(archbasefw) 878 879 # copy arch-independent files from last build into the basedir framework 880 basefw = os.path.join(basedir, *FW_VERSION_PREFIX) 881 shutil.copytree( 882 os.path.join(archbasefw, "include", "openssl"), 883 os.path.join(basefw, "include", "openssl") 884 ) 885 886 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"), 887 "SHLIB_VERSION_NUMBER") 888 # e.g. -> "1.0.0" 889 libcrypto = "libcrypto.dylib" 890 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".") 891 # e.g. -> "libcrypto.1.0.0.dylib" 892 libssl = "libssl.dylib" 893 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".") 894 # e.g. -> "libssl.1.0.0.dylib" 895 896 try: 897 os.mkdir(os.path.join(basefw, "lib")) 898 except OSError: 899 pass 900 901 # merge the individual arch-dependent shared libs into a fat shared lib 902 archbasefws.insert(0, basefw) 903 for (lib_unversioned, lib_versioned) in [ 904 (libcrypto, libcrypto_versioned), 905 (libssl, libssl_versioned) 906 ]: 907 runCommand("lipo -create -output " + 908 " ".join(shellQuote( 909 os.path.join(fw, "lib", lib_versioned)) 910 for fw in archbasefws)) 911 # and create an unversioned symlink of it 912 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned)) 913 914 # Create links in the temp include and lib dirs that will be injected 915 # into the Python build so that setup.py can find them while building 916 # and the versioned links so that the setup.py post-build import test 917 # does not fail. 918 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX) 919 for fn in [ 920 ["include", "openssl"], 921 ["lib", libcrypto], 922 ["lib", libssl], 923 ["lib", libcrypto_versioned], 924 ["lib", libssl_versioned], 925 ]: 926 os.symlink( 927 os.path.join(relative_path, *fn), 928 os.path.join(basedir, "usr", "local", *fn) 929 ) 930 931 return 932 933def buildRecipe(recipe, basedir, archList): 934 """ 935 Build software using a recipe. This function does the 936 'configure;make;make install' dance for C software, with a possibility 937 to customize this process, basically a poor-mans DarwinPorts. 938 """ 939 curdir = os.getcwd() 940 941 name = recipe['name'] 942 THIRD_PARTY_LIBS.append(name) 943 url = recipe['url'] 944 configure = recipe.get('configure', './configure') 945 buildrecipe = recipe.get('buildrecipe', None) 946 install = recipe.get('install', 'make && make install DESTDIR=%s'%( 947 shellQuote(basedir))) 948 949 archiveName = os.path.split(url)[-1] 950 sourceArchive = os.path.join(DEPSRC, archiveName) 951 952 if not os.path.exists(DEPSRC): 953 os.mkdir(DEPSRC) 954 955 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive) 956 print("Extracting archive for %s"%(name,)) 957 buildDir=os.path.join(WORKDIR, '_bld') 958 if not os.path.exists(buildDir): 959 os.mkdir(buildDir) 960 961 workDir = extractArchive(buildDir, sourceArchive) 962 os.chdir(workDir) 963 964 for patch in recipe.get('patches', ()): 965 if isinstance(patch, tuple): 966 url, checksum = patch 967 fn = os.path.join(DEPSRC, os.path.basename(url)) 968 verifyThirdPartyFile(url, checksum, fn) 969 else: 970 # patch is a file in the source directory 971 fn = os.path.join(curdir, patch) 972 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1), 973 shellQuote(fn),)) 974 975 for patchscript in recipe.get('patchscripts', ()): 976 if isinstance(patchscript, tuple): 977 url, checksum = patchscript 978 fn = os.path.join(DEPSRC, os.path.basename(url)) 979 verifyThirdPartyFile(url, checksum, fn) 980 else: 981 # patch is a file in the source directory 982 fn = os.path.join(curdir, patchscript) 983 if fn.endswith('.bz2'): 984 runCommand('bunzip2 -fk %s' % shellQuote(fn)) 985 fn = fn[:-4] 986 runCommand('sh %s' % shellQuote(fn)) 987 os.unlink(fn) 988 989 if 'buildDir' in recipe: 990 os.chdir(recipe['buildDir']) 991 992 if configure is not None: 993 configure_args = [ 994 "--prefix=/usr/local", 995 "--enable-static", 996 "--disable-shared", 997 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),), 998 ] 999 1000 if 'configure_pre' in recipe: 1001 args = list(recipe['configure_pre']) 1002 if '--disable-static' in args: 1003 configure_args.remove('--enable-static') 1004 if '--enable-shared' in args: 1005 configure_args.remove('--disable-shared') 1006 configure_args.extend(args) 1007 1008 if recipe.get('useLDFlags', 1): 1009 configure_args.extend([ 1010 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1011 "-I%s/usr/local/include"%( 1012 recipe.get('extra_cflags', ''), 1013 DEPTARGET, 1014 ' -arch '.join(archList), 1015 shellQuote(basedir)[1:-1],), 1016 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( 1017 DEPTARGET, 1018 shellQuote(basedir)[1:-1], 1019 ' -arch '.join(archList)), 1020 ]) 1021 else: 1022 configure_args.extend([ 1023 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1024 "-I%s/usr/local/include"%( 1025 recipe.get('extra_cflags', ''), 1026 DEPTARGET, 1027 ' -arch '.join(archList), 1028 shellQuote(basedir)[1:-1],), 1029 ]) 1030 1031 if 'configure_post' in recipe: 1032 configure_args = configure_args + list(recipe['configure_post']) 1033 1034 configure_args.insert(0, configure) 1035 configure_args = [ shellQuote(a) for a in configure_args ] 1036 1037 print("Running configure for %s"%(name,)) 1038 runCommand(' '.join(configure_args) + ' 2>&1') 1039 1040 if buildrecipe is not None: 1041 # call special-case build recipe, e.g. for openssl 1042 buildrecipe(basedir, archList) 1043 1044 if install is not None: 1045 print("Running install for %s"%(name,)) 1046 runCommand('{ ' + install + ' ;} 2>&1') 1047 1048 print("Done %s"%(name,)) 1049 print("") 1050 1051 os.chdir(curdir) 1052 1053def buildLibraries(): 1054 """ 1055 Build our dependencies into $WORKDIR/libraries/usr/local 1056 """ 1057 print("") 1058 print("Building required libraries") 1059 print("") 1060 universal = os.path.join(WORKDIR, 'libraries') 1061 os.mkdir(universal) 1062 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib')) 1063 os.makedirs(os.path.join(universal, 'usr', 'local', 'include')) 1064 1065 for recipe in library_recipes(): 1066 buildRecipe(recipe, universal, ARCHLIST) 1067 1068 1069 1070def buildPythonDocs(): 1071 # This stores the documentation as Resources/English.lproj/Documentation 1072 # inside the framework. pydoc and IDLE will pick it up there. 1073 print("Install python documentation") 1074 rootDir = os.path.join(WORKDIR, '_root') 1075 buildDir = os.path.join('../../Doc') 1076 docdir = os.path.join(rootDir, 'pydocs') 1077 curDir = os.getcwd() 1078 os.chdir(buildDir) 1079 runCommand('make clean') 1080 1081 # Search third-party source directory for a pre-built version of the docs. 1082 # Use the naming convention of the docs.python.org html downloads: 1083 # python-3.9.0b1-docs-html.tar.bz2 1084 doctarfiles = [ f for f in os.listdir(DEPSRC) 1085 if f.startswith('python-'+getFullVersion()) 1086 if f.endswith('-docs-html.tar.bz2') ] 1087 if doctarfiles: 1088 doctarfile = doctarfiles[0] 1089 if not os.path.exists('build'): 1090 os.mkdir('build') 1091 # if build directory existed, it was emptied by make clean, above 1092 os.chdir('build') 1093 # Extract the first archive found for this version into build 1094 runCommand('tar xjf %s'%shellQuote(os.path.join(DEPSRC, doctarfile))) 1095 # see if tar extracted a directory ending in -docs-html 1096 archivefiles = [ f for f in os.listdir('.') 1097 if f.endswith('-docs-html') 1098 if os.path.isdir(f) ] 1099 if archivefiles: 1100 archivefile = archivefiles[0] 1101 # make it our 'Docs/build/html' directory 1102 print(' -- using pre-built python documentation from %s'%archivefile) 1103 os.rename(archivefile, 'html') 1104 os.chdir(buildDir) 1105 1106 htmlDir = os.path.join('build', 'html') 1107 if not os.path.exists(htmlDir): 1108 # Create virtual environment for docs builds with blurb and sphinx 1109 runCommand('make venv') 1110 runCommand('make html PYTHON=venv/bin/python') 1111 os.rename(htmlDir, docdir) 1112 os.chdir(curDir) 1113 1114 1115def buildPython(): 1116 print("Building a universal python for %s architectures" % UNIVERSALARCHS) 1117 1118 buildDir = os.path.join(WORKDIR, '_bld', 'python') 1119 rootDir = os.path.join(WORKDIR, '_root') 1120 1121 if os.path.exists(buildDir): 1122 shutil.rmtree(buildDir) 1123 if os.path.exists(rootDir): 1124 shutil.rmtree(rootDir) 1125 os.makedirs(buildDir) 1126 os.makedirs(rootDir) 1127 os.makedirs(os.path.join(rootDir, 'empty-dir')) 1128 curdir = os.getcwd() 1129 os.chdir(buildDir) 1130 1131 # Extract the version from the configure file, needed to calculate 1132 # several paths. 1133 version = getVersion() 1134 1135 # Since the extra libs are not in their installed framework location 1136 # during the build, augment the library path so that the interpreter 1137 # will find them during its extension import sanity checks. 1138 1139 print("Running configure...") 1140 runCommand("%s -C --enable-framework --enable-universalsdk=/ " 1141 "--with-universal-archs=%s " 1142 "%s " 1143 "%s " 1144 "%s " 1145 "%s " 1146 "%s " 1147 "%s " 1148 "LDFLAGS='-g -L%s/libraries/usr/local/lib' " 1149 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( 1150 shellQuote(os.path.join(SRCDIR, 'configure')), 1151 UNIVERSALARCHS, 1152 (' ', '--with-computed-gotos ')[PYTHON_3], 1153 (' ', '--without-ensurepip ')[PYTHON_3], 1154 (' ', "--with-openssl='%s/libraries/usr/local'"%( 1155 shellQuote(WORKDIR)[1:-1],))[PYTHON_3], 1156 (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( 1157 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1158 (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( 1159 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1160 (' ', "--enable-optimizations --with-lto")[compilerCanOptimize()], 1161 shellQuote(WORKDIR)[1:-1], 1162 shellQuote(WORKDIR)[1:-1])) 1163 1164 # As of macOS 10.11 with SYSTEM INTEGRITY PROTECTION, DYLD_* 1165 # environment variables are no longer automatically inherited 1166 # by child processes from their parents. We used to just set 1167 # DYLD_LIBRARY_PATH, pointing to the third-party libs, 1168 # in build-installer.py's process environment and it was 1169 # passed through the make utility into the environment of 1170 # setup.py. Instead, we now append DYLD_LIBRARY_PATH to 1171 # the existing RUNSHARED configuration value when we call 1172 # make for extension module builds. 1173 1174 runshared_for_make = "".join([ 1175 " RUNSHARED=", 1176 "'", 1177 grepValue("Makefile", "RUNSHARED"), 1178 ' DYLD_LIBRARY_PATH=', 1179 os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib'), 1180 "'" ]) 1181 1182 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS 1183 # and, if defined, append its value to the make command. This allows 1184 # us to pass in version control tags, like GITTAG, to a build from a 1185 # tarball rather than from a vcs checkout, thus eliminating the need 1186 # to have a working copy of the vcs program on the build machine. 1187 # 1188 # A typical use might be: 1189 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ 1190 # GITVERSION='echo 123456789a' \ 1191 # GITTAG='echo v3.6.0' \ 1192 # GITBRANCH='echo 3.6'" 1193 1194 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") 1195 if make_extras: 1196 make_cmd = "make " + make_extras + runshared_for_make 1197 else: 1198 make_cmd = "make" + runshared_for_make 1199 print("Running " + make_cmd) 1200 runCommand(make_cmd) 1201 1202 make_cmd = "make install DESTDIR=%s %s"%( 1203 shellQuote(rootDir), 1204 runshared_for_make) 1205 print("Running " + make_cmd) 1206 runCommand(make_cmd) 1207 1208 make_cmd = "make frameworkinstallextras DESTDIR=%s %s"%( 1209 shellQuote(rootDir), 1210 runshared_for_make) 1211 print("Running " + make_cmd) 1212 runCommand(make_cmd) 1213 1214 print("Copying required shared libraries") 1215 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): 1216 build_lib_dir = os.path.join( 1217 WORKDIR, 'libraries', 'Library', 'Frameworks', 1218 'Python.framework', 'Versions', getVersion(), 'lib') 1219 fw_lib_dir = os.path.join( 1220 WORKDIR, '_root', 'Library', 'Frameworks', 1221 'Python.framework', 'Versions', getVersion(), 'lib') 1222 if internalTk(): 1223 # move Tcl and Tk pkgconfig files 1224 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( 1225 shellQuote(build_lib_dir), 1226 shellQuote(fw_lib_dir) )) 1227 runCommand("rm -r %s/pkgconfig"%( 1228 shellQuote(build_lib_dir), )) 1229 runCommand("mv %s/* %s"%( 1230 shellQuote(build_lib_dir), 1231 shellQuote(fw_lib_dir) )) 1232 1233 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') 1234 frmDirVersioned = os.path.join(frmDir, 'Versions', version) 1235 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,)) 1236 # create directory for OpenSSL certificates 1237 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl') 1238 os.makedirs(sslDir) 1239 1240 print("Fix file modes") 1241 gid = grp.getgrnam('admin').gr_gid 1242 1243 shared_lib_error = False 1244 for dirpath, dirnames, filenames in os.walk(frmDir): 1245 for dn in dirnames: 1246 os.chmod(os.path.join(dirpath, dn), STAT_0o775) 1247 os.chown(os.path.join(dirpath, dn), -1, gid) 1248 1249 for fn in filenames: 1250 if os.path.islink(fn): 1251 continue 1252 1253 # "chmod g+w $fn" 1254 p = os.path.join(dirpath, fn) 1255 st = os.stat(p) 1256 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP) 1257 os.chown(p, -1, gid) 1258 1259 if fn in EXPECTED_SHARED_LIBS: 1260 # check to see that this file was linked with the 1261 # expected library path and version 1262 data = captureCommand("otool -L %s" % shellQuote(p)) 1263 for sl in EXPECTED_SHARED_LIBS[fn]: 1264 if ("\t%s " % sl) not in data: 1265 print("Expected shared lib %s was not linked with %s" 1266 % (sl, p)) 1267 shared_lib_error = True 1268 1269 if shared_lib_error: 1270 fatal("Unexpected shared library errors.") 1271 1272 if PYTHON_3: 1273 LDVERSION=None 1274 VERSION=None 1275 ABIFLAGS=None 1276 1277 fp = open(os.path.join(buildDir, 'Makefile'), 'r') 1278 for ln in fp: 1279 if ln.startswith('VERSION='): 1280 VERSION=ln.split()[1] 1281 if ln.startswith('ABIFLAGS='): 1282 ABIFLAGS=ln.split() 1283 ABIFLAGS=ABIFLAGS[1] if len(ABIFLAGS) > 1 else '' 1284 if ln.startswith('LDVERSION='): 1285 LDVERSION=ln.split()[1] 1286 fp.close() 1287 1288 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION) 1289 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS) 1290 config_suffix = '-' + LDVERSION 1291 if getVersionMajorMinor() >= (3, 6): 1292 config_suffix = config_suffix + '-darwin' 1293 else: 1294 config_suffix = '' # Python 2.x 1295 1296 # We added some directories to the search path during the configure 1297 # phase. Remove those because those directories won't be there on 1298 # the end-users system. Also remove the directories from _sysconfigdata.py 1299 # (added in 3.3) if it exists. 1300 1301 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,) 1302 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,) 1303 1304 # fix Makefile 1305 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile') 1306 fp = open(path, 'r') 1307 data = fp.read() 1308 fp.close() 1309 1310 for p in (include_path, lib_path): 1311 data = data.replace(" " + p, '') 1312 data = data.replace(p + " ", '') 1313 1314 fp = open(path, 'w') 1315 fp.write(data) 1316 fp.close() 1317 1318 # fix _sysconfigdata 1319 # 1320 # TODO: make this more robust! test_sysconfig_module of 1321 # distutils.tests.test_sysconfig.SysconfigTestCase tests that 1322 # the output from get_config_var in both sysconfig and 1323 # distutils.sysconfig is exactly the same for both CFLAGS and 1324 # LDFLAGS. The fixing up is now complicated by the pretty 1325 # printing in _sysconfigdata.py. Also, we are using the 1326 # pprint from the Python running the installer build which 1327 # may not cosmetically format the same as the pprint in the Python 1328 # being built (and which is used to originally generate 1329 # _sysconfigdata.py). 1330 1331 import pprint 1332 if getVersionMajorMinor() >= (3, 6): 1333 # XXX this is extra-fragile 1334 path = os.path.join(path_to_lib, 1335 '_sysconfigdata_%s_darwin_darwin.py' % (ABIFLAGS,)) 1336 else: 1337 path = os.path.join(path_to_lib, '_sysconfigdata.py') 1338 fp = open(path, 'r') 1339 data = fp.read() 1340 fp.close() 1341 # create build_time_vars dict 1342 if RUNNING_ON_PYTHON2: 1343 exec(data) 1344 else: 1345 g_dict = {} 1346 l_dict = {} 1347 exec(data, g_dict, l_dict) 1348 build_time_vars = l_dict['build_time_vars'] 1349 vars = {} 1350 for k, v in build_time_vars.items(): 1351 if type(v) == type(''): 1352 for p in (include_path, lib_path): 1353 v = v.replace(' ' + p, '') 1354 v = v.replace(p + ' ', '') 1355 vars[k] = v 1356 1357 fp = open(path, 'w') 1358 # duplicated from sysconfig._generate_posix_vars() 1359 fp.write('# system configuration generated and used by' 1360 ' the sysconfig module\n') 1361 fp.write('build_time_vars = ') 1362 pprint.pprint(vars, stream=fp) 1363 fp.close() 1364 1365 # Add symlinks in /usr/local/bin, using relative links 1366 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin') 1367 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks', 1368 'Python.framework', 'Versions', version, 'bin') 1369 if os.path.exists(usr_local_bin): 1370 shutil.rmtree(usr_local_bin) 1371 os.makedirs(usr_local_bin) 1372 for fn in os.listdir( 1373 os.path.join(frmDir, 'Versions', version, 'bin')): 1374 os.symlink(os.path.join(to_framework, fn), 1375 os.path.join(usr_local_bin, fn)) 1376 1377 os.chdir(curdir) 1378 1379def patchFile(inPath, outPath): 1380 data = fileContents(inPath) 1381 data = data.replace('$FULL_VERSION', getFullVersion()) 1382 data = data.replace('$VERSION', getVersion()) 1383 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later'))) 1384 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS])) 1385 data = data.replace('$INSTALL_SIZE', installSize()) 1386 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS)) 1387 1388 # This one is not handy as a template variable 1389 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework') 1390 fp = open(outPath, 'w') 1391 fp.write(data) 1392 fp.close() 1393 1394def patchScript(inPath, outPath): 1395 major, minor = getVersionMajorMinor() 1396 data = fileContents(inPath) 1397 data = data.replace('@PYMAJOR@', str(major)) 1398 data = data.replace('@PYVER@', getVersion()) 1399 fp = open(outPath, 'w') 1400 fp.write(data) 1401 fp.close() 1402 os.chmod(outPath, STAT_0o755) 1403 1404 1405 1406def packageFromRecipe(targetDir, recipe): 1407 curdir = os.getcwd() 1408 try: 1409 # The major version (such as 2.5) is included in the package name 1410 # because having two version of python installed at the same time is 1411 # common. 1412 pkgname = '%s-%s'%(recipe['name'], getVersion()) 1413 srcdir = recipe.get('source') 1414 pkgroot = recipe.get('topdir', srcdir) 1415 postflight = recipe.get('postflight') 1416 readme = textwrap.dedent(recipe['readme']) 1417 isRequired = recipe.get('required', True) 1418 1419 print("- building package %s"%(pkgname,)) 1420 1421 # Substitute some variables 1422 textvars = dict( 1423 VER=getVersion(), 1424 FULLVER=getFullVersion(), 1425 ) 1426 readme = readme % textvars 1427 1428 if pkgroot is not None: 1429 pkgroot = pkgroot % textvars 1430 else: 1431 pkgroot = '/' 1432 1433 if srcdir is not None: 1434 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:]) 1435 srcdir = srcdir % textvars 1436 1437 if postflight is not None: 1438 postflight = os.path.abspath(postflight) 1439 1440 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents') 1441 os.makedirs(packageContents) 1442 1443 if srcdir is not None: 1444 os.chdir(srcdir) 1445 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1446 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1447 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),)) 1448 1449 fn = os.path.join(packageContents, 'PkgInfo') 1450 fp = open(fn, 'w') 1451 fp.write('pmkrpkg1') 1452 fp.close() 1453 1454 rsrcDir = os.path.join(packageContents, "Resources") 1455 os.mkdir(rsrcDir) 1456 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w') 1457 fp.write(readme) 1458 fp.close() 1459 1460 if postflight is not None: 1461 patchScript(postflight, os.path.join(rsrcDir, 'postflight')) 1462 1463 vers = getFullVersion() 1464 major, minor = getVersionMajorMinor() 1465 pl = dict( 1466 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), 1467 CFBundleIdentifier='org.python.Python.%s'%(pkgname,), 1468 CFBundleName='Python.%s'%(pkgname,), 1469 CFBundleShortVersionString=vers, 1470 IFMajorVersion=major, 1471 IFMinorVersion=minor, 1472 IFPkgFormatVersion=0.10000000149011612, 1473 IFPkgFlagAllowBackRev=False, 1474 IFPkgFlagAuthorizationAction="RootAuthorization", 1475 IFPkgFlagDefaultLocation=pkgroot, 1476 IFPkgFlagFollowLinks=True, 1477 IFPkgFlagInstallFat=True, 1478 IFPkgFlagIsRequired=isRequired, 1479 IFPkgFlagOverwritePermissions=False, 1480 IFPkgFlagRelocatable=False, 1481 IFPkgFlagRestartAction="NoRestart", 1482 IFPkgFlagRootVolumeOnly=True, 1483 IFPkgFlagUpdateInstalledLangauges=False, 1484 ) 1485 writePlist(pl, os.path.join(packageContents, 'Info.plist')) 1486 1487 pl = dict( 1488 IFPkgDescriptionDescription=readme, 1489 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), 1490 IFPkgDescriptionVersion=vers, 1491 ) 1492 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist')) 1493 1494 finally: 1495 os.chdir(curdir) 1496 1497 1498def makeMpkgPlist(path): 1499 1500 vers = getFullVersion() 1501 major, minor = getVersionMajorMinor() 1502 1503 pl = dict( 1504 CFBundleGetInfoString="Python %s"%(vers,), 1505 CFBundleIdentifier='org.python.Python', 1506 CFBundleName='Python', 1507 CFBundleShortVersionString=vers, 1508 IFMajorVersion=major, 1509 IFMinorVersion=minor, 1510 IFPkgFlagComponentDirectory="Contents/Packages", 1511 IFPkgFlagPackageList=[ 1512 dict( 1513 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()), 1514 IFPkgFlagPackageSelection=item.get('selected', 'selected'), 1515 ) 1516 for item in pkg_recipes() 1517 ], 1518 IFPkgFormatVersion=0.10000000149011612, 1519 IFPkgFlagBackgroundScaling="proportional", 1520 IFPkgFlagBackgroundAlignment="left", 1521 IFPkgFlagAuthorizationAction="RootAuthorization", 1522 ) 1523 1524 writePlist(pl, path) 1525 1526 1527def buildInstaller(): 1528 1529 # Zap all compiled files 1530 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')): 1531 for fn in filenames: 1532 if fn.endswith('.pyc') or fn.endswith('.pyo'): 1533 os.unlink(os.path.join(dirpath, fn)) 1534 1535 outdir = os.path.join(WORKDIR, 'installer') 1536 if os.path.exists(outdir): 1537 shutil.rmtree(outdir) 1538 os.mkdir(outdir) 1539 1540 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents') 1541 pkgcontents = os.path.join(pkgroot, 'Packages') 1542 os.makedirs(pkgcontents) 1543 for recipe in pkg_recipes(): 1544 packageFromRecipe(pkgcontents, recipe) 1545 1546 rsrcDir = os.path.join(pkgroot, 'Resources') 1547 1548 fn = os.path.join(pkgroot, 'PkgInfo') 1549 fp = open(fn, 'w') 1550 fp.write('pmkrpkg1') 1551 fp.close() 1552 1553 os.mkdir(rsrcDir) 1554 1555 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) 1556 pl = dict( 1557 IFPkgDescriptionTitle="Python", 1558 IFPkgDescriptionVersion=getVersion(), 1559 ) 1560 1561 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist')) 1562 for fn in os.listdir('resources'): 1563 if fn == '.svn': continue 1564 if fn.endswith('.jpg'): 1565 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1566 else: 1567 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1568 1569 1570def installSize(clear=False, _saved=[]): 1571 if clear: 1572 del _saved[:] 1573 if not _saved: 1574 data = captureCommand("du -ks %s"%( 1575 shellQuote(os.path.join(WORKDIR, '_root')))) 1576 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),)) 1577 return _saved[0] 1578 1579 1580def buildDMG(): 1581 """ 1582 Create DMG containing the rootDir. 1583 """ 1584 outdir = os.path.join(WORKDIR, 'diskimage') 1585 if os.path.exists(outdir): 1586 shutil.rmtree(outdir) 1587 1588 # We used to use the deployment target as the last characters of the 1589 # installer file name. With the introduction of weaklinked installer 1590 # variants, we may have two variants with the same file name, i.e. 1591 # both ending in '10.9'. To avoid this, we now use the major/minor 1592 # version numbers of the macOS version we are building on. 1593 # Also, as of macOS 11, operating system version numbering has 1594 # changed from three components to two, i.e. 1595 # 10.14.1, 10.14.2, ... 1596 # 10.15.1, 10.15.2, ... 1597 # 11.1, 11.2, ... 1598 # 12.1, 12.2, ... 1599 # (A further twist is that, when running on macOS 11, binaries built 1600 # on older systems may be shown an operating system version of 10.16 1601 # instead of 11. We should not run into that situation here.) 1602 # Also we should use "macos" instead of "macosx" going forward. 1603 # 1604 # To maintain compatibility for legacy variants, the file name for 1605 # builds on macOS 10.15 and earlier remains: 1606 # python-3.x.y-macosx10.z.{dmg->pkg} 1607 # e.g. python-3.9.4-macosx10.9.{dmg->pkg} 1608 # and for builds on macOS 11+: 1609 # python-3.x.y-macosz.{dmg->pkg} 1610 # e.g. python-3.9.4-macos11.{dmg->pkg} 1611 1612 build_tuple = getBuildTuple() 1613 if build_tuple[0] < 11: 1614 os_name = 'macosx' 1615 build_system_version = '%s.%s' % build_tuple 1616 else: 1617 os_name = 'macos' 1618 build_system_version = str(build_tuple[0]) 1619 imagepath = os.path.join(outdir, 1620 'python-%s-%s%s'%(getFullVersion(),os_name,build_system_version)) 1621 if INCLUDE_TIMESTAMP: 1622 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3]) 1623 imagepath = imagepath + '.dmg' 1624 1625 os.mkdir(outdir) 1626 1627 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9, 1628 # when hdiutil create fails with "Resource busy". For now, just retry 1629 # the create a few times and hope that it eventually works. 1630 1631 volname='Python %s'%(getFullVersion()) 1632 cmd = ("hdiutil create -format UDRW -volname %s -srcfolder %s -size 100m %s"%( 1633 shellQuote(volname), 1634 shellQuote(os.path.join(WORKDIR, 'installer')), 1635 shellQuote(imagepath + ".tmp.dmg" ))) 1636 for i in range(5): 1637 fd = os.popen(cmd, 'r') 1638 data = fd.read() 1639 xit = fd.close() 1640 if not xit: 1641 break 1642 sys.stdout.write(data) 1643 print(" -- retrying hdiutil create") 1644 time.sleep(5) 1645 else: 1646 raise RuntimeError("command failed: %s"%(cmd,)) 1647 1648 if not os.path.exists(os.path.join(WORKDIR, "mnt")): 1649 os.mkdir(os.path.join(WORKDIR, "mnt")) 1650 runCommand("hdiutil attach %s -mountroot %s"%( 1651 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt")))) 1652 1653 # Custom icon for the DMG, shown when the DMG is mounted. 1654 shutil.copy("../Icons/Disk Image.icns", 1655 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns")) 1656 runCommand("SetFile -a C %s/"%( 1657 shellQuote(os.path.join(WORKDIR, "mnt", volname)),)) 1658 1659 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname)))) 1660 1661 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns") 1662 runCommand("hdiutil convert %s -format UDZO -o %s"%( 1663 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath))) 1664 setIcon(imagepath, "../Icons/Disk Image.icns") 1665 1666 os.unlink(imagepath + ".tmp.dmg") 1667 1668 return imagepath 1669 1670 1671def setIcon(filePath, icnsPath): 1672 """ 1673 Set the custom icon for the specified file or directory. 1674 """ 1675 1676 dirPath = os.path.normpath(os.path.dirname(__file__)) 1677 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon") 1678 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime: 1679 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due 1680 # to connections to the window server. 1681 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS") 1682 if not os.path.exists(appPath): 1683 os.makedirs(appPath) 1684 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%( 1685 shellQuote(toolPath), shellQuote(dirPath))) 1686 1687 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath), 1688 shellQuote(filePath))) 1689 1690def main(): 1691 # First parse options and check if we can perform our work 1692 parseOptions() 1693 checkEnvironment() 1694 1695 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET 1696 os.environ['CC'] = CC 1697 os.environ['CXX'] = CXX 1698 1699 if os.path.exists(WORKDIR): 1700 shutil.rmtree(WORKDIR) 1701 os.mkdir(WORKDIR) 1702 1703 os.environ['LC_ALL'] = 'C' 1704 1705 # Then build third-party libraries such as sleepycat DB4. 1706 buildLibraries() 1707 1708 # Now build python itself 1709 buildPython() 1710 1711 # And then build the documentation 1712 # Remove the Deployment Target from the shell 1713 # environment, it's no longer needed and 1714 # an unexpected build target can cause problems 1715 # when Sphinx and its dependencies need to 1716 # be (re-)installed. 1717 del os.environ['MACOSX_DEPLOYMENT_TARGET'] 1718 buildPythonDocs() 1719 1720 1721 # Prepare the applications folder 1722 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%( 1723 getVersion(),)) 1724 fn = os.path.join(folder, "License.rtf") 1725 patchFile("resources/License.rtf", fn) 1726 fn = os.path.join(folder, "ReadMe.rtf") 1727 patchFile("resources/ReadMe.rtf", fn) 1728 fn = os.path.join(folder, "Update Shell Profile.command") 1729 patchScript("scripts/postflight.patch-profile", fn) 1730 fn = os.path.join(folder, "Install Certificates.command") 1731 patchScript("resources/install_certificates.command", fn) 1732 os.chmod(folder, STAT_0o755) 1733 setIcon(folder, "../Icons/Python Folder.icns") 1734 1735 # Create the installer 1736 buildInstaller() 1737 1738 # And copy the readme into the directory containing the installer 1739 patchFile('resources/ReadMe.rtf', 1740 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf')) 1741 1742 # Ditto for the license file. 1743 patchFile('resources/License.rtf', 1744 os.path.join(WORKDIR, 'installer', 'License.rtf')) 1745 1746 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w') 1747 fp.write("# BUILD INFO\n") 1748 fp.write("# Date: %s\n" % time.ctime()) 1749 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos) 1750 fp.close() 1751 1752 # And copy it to a DMG 1753 buildDMG() 1754 1755if __name__ == "__main__": 1756 main() 1757