1#!/usr/bin/env python 2""" 3This script is used to build "official" universal installers on macOS. 4 5NEW for 3.10 and backports: 6- support universal2 variant with arm64 and x86_64 archs 7- enable clang optimizations when building on 10.15+ 8 9NEW for 3.9.0 and backports: 10- 2.7 end-of-life issues: 11 - Python 3 installs now update the Current version link 12 in /Library/Frameworks/Python.framework/Versions 13- fully support running under Python 3 as well as 2.7 14- support building on newer macOS systems with SIP 15- fully support building on macOS 10.9+ 16- support 10.6+ on best effort 17- support bypassing docs build by supplying a prebuilt 18 docs html tarball in the third-party source library, 19 in the format and filename conventional of those 20 downloadable from python.org: 21 python-3.x.y-docs-html.tar.bz2 22 23NEW for 3.7.0: 24- support Intel 64-bit-only () and 32-bit-only installer builds 25- build and use internal Tcl/Tk 8.6 for 10.6+ builds 26- deprecate use of explicit SDK (--sdk-path=) since all but the oldest 27 versions of Xcode support implicit setting of an SDK via environment 28 variables (SDKROOT and friends, see the xcrun man page for more info). 29 The SDK stuff was primarily needed for building universal installers 30 for 10.4; so as of 3.7.0, building installers for 10.4 is no longer 31 supported with build-installer. 32- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" 33 34TODO: 35- test building with SDKROOT and DEVELOPER_DIR xcrun env variables 36 37Usage: see USAGE variable in the script. 38""" 39import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp 40try: 41 import urllib2 as urllib_request 42except ImportError: 43 import urllib.request as urllib_request 44 45STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 46 | stat.S_IRGRP | stat.S_IXGRP 47 | stat.S_IROTH | stat.S_IXOTH ) 48 49STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 50 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP 51 | stat.S_IROTH | stat.S_IXOTH ) 52 53INCLUDE_TIMESTAMP = 1 54VERBOSE = 1 55 56RUNNING_ON_PYTHON2 = sys.version_info.major == 2 57 58if RUNNING_ON_PYTHON2: 59 from plistlib import writePlist 60else: 61 from plistlib import dump 62 def writePlist(path, plist): 63 with open(plist, 'wb') as fp: 64 dump(path, fp) 65 66def shellQuote(value): 67 """ 68 Return the string value in a form that can safely be inserted into 69 a shell command. 70 """ 71 return "'%s'"%(value.replace("'", "'\"'\"'")) 72 73def grepValue(fn, variable): 74 """ 75 Return the unquoted value of a variable from a file.. 76 QUOTED_VALUE='quotes' -> str('quotes') 77 UNQUOTED_VALUE=noquotes -> str('noquotes') 78 """ 79 variable = variable + '=' 80 for ln in open(fn, 'r'): 81 if ln.startswith(variable): 82 value = ln[len(variable):].strip() 83 return value.strip("\"'") 84 raise RuntimeError("Cannot find variable %s" % variable[:-1]) 85 86_cache_getVersion = None 87 88def getVersion(): 89 global _cache_getVersion 90 if _cache_getVersion is None: 91 _cache_getVersion = grepValue( 92 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION') 93 return _cache_getVersion 94 95def getVersionMajorMinor(): 96 return tuple([int(n) for n in getVersion().split('.', 2)]) 97 98_cache_getFullVersion = None 99 100def getFullVersion(): 101 global _cache_getFullVersion 102 if _cache_getFullVersion is not None: 103 return _cache_getFullVersion 104 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h') 105 for ln in open(fn): 106 if 'PY_VERSION' in ln: 107 _cache_getFullVersion = ln.split()[-1][1:-1] 108 return _cache_getFullVersion 109 raise RuntimeError("Cannot find full version??") 110 111FW_PREFIX = ["Library", "Frameworks", "Python.framework"] 112FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions 113FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions 114 115# The directory we'll use to create the build (will be erased and recreated) 116WORKDIR = "/tmp/_py" 117 118# The directory we'll use to store third-party sources. Set this to something 119# else if you don't want to re-fetch required libraries every time. 120DEPSRC = os.path.join(WORKDIR, 'third-party') 121DEPSRC = os.path.expanduser('~/Universal/other-sources') 122 123universal_opts_map = { 'universal2': ('arm64', 'x86_64'), 124 '32-bit': ('i386', 'ppc',), 125 '64-bit': ('x86_64', 'ppc64',), 126 'intel': ('i386', 'x86_64'), 127 'intel-32': ('i386',), 128 'intel-64': ('x86_64',), 129 '3-way': ('ppc', 'i386', 'x86_64'), 130 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } 131default_target_map = { 132 'universal2': '10.9', 133 '64-bit': '10.5', 134 '3-way': '10.5', 135 'intel': '10.5', 136 'intel-32': '10.4', 137 'intel-64': '10.5', 138 'all': '10.5', 139} 140 141UNIVERSALOPTS = tuple(universal_opts_map.keys()) 142 143UNIVERSALARCHS = '32-bit' 144 145ARCHLIST = universal_opts_map[UNIVERSALARCHS] 146 147# Source directory (assume we're in Mac/BuildScript) 148SRCDIR = os.path.dirname( 149 os.path.dirname( 150 os.path.dirname( 151 os.path.abspath(__file__ 152 )))) 153 154# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level 155DEPTARGET = '10.5' 156 157def getDeptargetTuple(): 158 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) 159 160def getBuildTuple(): 161 return tuple([int(n) for n in platform.mac_ver()[0].split('.')[0:2]]) 162 163def getTargetCompilers(): 164 target_cc_map = { 165 '10.4': ('gcc-4.0', 'g++-4.0'), 166 '10.5': ('gcc', 'g++'), 167 '10.6': ('gcc', 'g++'), 168 '10.7': ('gcc', 'g++'), 169 '10.8': ('gcc', 'g++'), 170 } 171 return target_cc_map.get(DEPTARGET, ('clang', 'clang++') ) 172 173CC, CXX = getTargetCompilers() 174 175PYTHON_3 = getVersionMajorMinor() >= (3, 0) 176 177USAGE = textwrap.dedent("""\ 178 Usage: build_python [options] 179 180 Options: 181 -? or -h: Show this message 182 -b DIR 183 --build-dir=DIR: Create build here (default: %(WORKDIR)r) 184 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) 185 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) 186 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) 187 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) 188 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) 189""")% globals() 190 191# Dict of object file names with shared library names to check after building. 192# This is to ensure that we ended up dynamically linking with the shared 193# library paths and versions we expected. For example: 194# EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 195# '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl', 196# '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] 197EXPECTED_SHARED_LIBS = {} 198 199# Are we building and linking with our own copy of Tcl/TK? 200# For now, do so if deployment target is 10.6+. 201def internalTk(): 202 return getDeptargetTuple() >= (10, 6) 203 204# Do we use 8.6.8 when building our own copy 205# of Tcl/Tk or a modern version. 206# We use the old version when building on 207# old versions of macOS due to build issues. 208def useOldTk(): 209 return getBuildTuple() < (10, 15) 210 211 212def tweak_tcl_build(basedir, archList): 213 with open("Makefile", "r") as fp: 214 contents = fp.readlines() 215 216 # For reasons I don't understand the tcl configure script 217 # decides that some stdlib symbols aren't present, before 218 # deciding that strtod is broken. 219 new_contents = [] 220 for line in contents: 221 if line.startswith("COMPAT_OBJS"): 222 # note: the space before strtod.o is intentional, 223 # the detection of a broken strtod results in 224 # "fixstrod.o" on this line. 225 for nm in ("strstr.o", "strtoul.o", " strtod.o"): 226 line = line.replace(nm, "") 227 new_contents.append(line) 228 229 with open("Makefile", "w") as fp: 230 fp.writelines(new_contents) 231 232# List of names of third party software built with this installer. 233# The names will be inserted into the rtf version of the License. 234THIRD_PARTY_LIBS = [] 235 236# Instructions for building libraries that are necessary for building a 237# batteries included python. 238# [The recipes are defined here for convenience but instantiated later after 239# command line options have been processed.] 240def library_recipes(): 241 result = [] 242 243 # Since Apple removed the header files for the deprecated system 244 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not 245 # have much choice but to build our own copy here, too. 246 247 result.extend([ 248 dict( 249 name="OpenSSL 3.0.15", 250 url="https://github.com/openssl/openssl/releases/download/openssl-3.0.15/openssl-3.0.15.tar.gz", 251 checksum='23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533', 252 buildrecipe=build_universal_openssl, 253 configure=None, 254 install=None, 255 ), 256 ]) 257 258 if internalTk(): 259 if useOldTk(): 260 tcl_tk_ver='8.6.8' 261 tcl_checksum='81656d3367af032e0ae6157eff134f89' 262 263 tk_checksum='5e0faecba458ee1386078fb228d008ba' 264 tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] 265 266 else: 267 tcl_tk_ver='8.6.15' 268 tcl_checksum='861e159753f2e2fbd6ec1484103715b0be56be3357522b858d3cbb5f893ffef1' 269 270 tk_checksum='550969f35379f952b3020f3ab7b9dd5bfd11c1ef7c9b7c6a75f5c49aca793fec' 271 tk_patches = [] 272 273 274 base_url = "https://prdownloads.sourceforge.net/tcl/{what}{version}-src.tar.gz" 275 result.extend([ 276 dict( 277 name="Tcl %s"%(tcl_tk_ver,), 278 url=base_url.format(what="tcl", version=tcl_tk_ver), 279 checksum=tcl_checksum, 280 buildDir="unix", 281 configure_pre=[ 282 '--enable-shared', 283 '--enable-threads', 284 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 285 ], 286 useLDFlags=False, 287 buildrecipe=tweak_tcl_build, 288 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 289 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 290 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 291 }, 292 ), 293 dict( 294 name="Tk %s"%(tcl_tk_ver,), 295 url=base_url.format(what="tk", version=tcl_tk_ver), 296 checksum=tk_checksum, 297 patches=tk_patches, 298 buildDir="unix", 299 configure_pre=[ 300 '--enable-aqua', 301 '--enable-shared', 302 '--enable-threads', 303 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 304 ], 305 useLDFlags=False, 306 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 307 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 308 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 309 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), 310 }, 311 ), 312 ]) 313 314 if PYTHON_3: 315 result.extend([ 316 dict( 317 name="XZ 5.2.3", 318 url="http://tukaani.org/xz/xz-5.2.3.tar.gz", 319 checksum='ef68674fb47a8b8e741b34e429d86e9d', 320 configure_pre=[ 321 '--disable-dependency-tracking', 322 ] 323 ), 324 ]) 325 326 result.extend([ 327 dict( 328 name="NCurses 5.9", 329 url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz", 330 checksum='8cb9c412e5f2d96bc6f459aa8c6282a1', 331 configure_pre=[ 332 "--enable-widec", 333 "--without-cxx", 334 "--without-cxx-binding", 335 "--without-ada", 336 "--without-curses-h", 337 "--enable-shared", 338 "--with-shared", 339 "--without-debug", 340 "--without-normal", 341 "--without-tests", 342 "--without-manpages", 343 "--datadir=/usr/share", 344 "--sysconfdir=/etc", 345 "--sharedstatedir=/usr/com", 346 "--with-terminfo-dirs=/usr/share/terminfo", 347 "--with-default-terminfo-dir=/usr/share/terminfo", 348 "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), 349 ], 350 patchscripts=[ 351 ("ftp://ftp.invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", 352 "f54bf02a349f96a7c4f0d00922f3a0d4"), 353 ], 354 useLDFlags=False, 355 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%( 356 shellQuote(os.path.join(WORKDIR, 'libraries')), 357 shellQuote(os.path.join(WORKDIR, 'libraries')), 358 getVersion(), 359 ), 360 ), 361 dict( 362 name="SQLite 3.45.3", 363 url="https://sqlite.org/2024/sqlite-autoconf-3450300.tar.gz", 364 checksum="b2809ca53124c19c60f42bf627736eae011afdcc205bb48270a5ee9a38191531", 365 extra_cflags=('-Os ' 366 '-DSQLITE_ENABLE_FTS5 ' 367 '-DSQLITE_ENABLE_FTS4 ' 368 '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' 369 '-DSQLITE_ENABLE_RTREE ' 370 '-DSQLITE_OMIT_AUTOINIT ' 371 '-DSQLITE_TCL=0 ' 372 ), 373 configure_pre=[ 374 '--enable-threadsafe', 375 '--enable-shared=no', 376 '--enable-static=yes', 377 '--disable-readline', 378 '--disable-dependency-tracking', 379 ] 380 ), 381 dict( 382 name="libmpdec 4.0.0", 383 url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", 384 checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", 385 configure_pre=[ 386 "--disable-cxx", 387 "MACHINE=universal", 388 ] 389 ), 390 ]) 391 392 if not PYTHON_3: 393 result.extend([ 394 dict( 395 name="Sleepycat DB 4.7.25", 396 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz", 397 checksum='ec2b87e833779681a0c3a814aa71359e', 398 buildDir="build_unix", 399 configure="../dist/configure", 400 configure_pre=[ 401 '--includedir=/usr/local/include/db4', 402 ] 403 ), 404 ]) 405 406 return result 407 408def compilerCanOptimize(): 409 """ 410 Return True iff the default Xcode version can use PGO and LTO 411 """ 412 # bpo-42235: The version check is pretty conservative, can be 413 # adjusted after testing 414 mac_ver = tuple(map(int, platform.mac_ver()[0].split('.'))) 415 return mac_ver >= (10, 15) 416 417# Instructions for building packages inside the .mpkg. 418def pkg_recipes(): 419 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3] 420 result = [ 421 dict( 422 name="PythonFramework", 423 long_name="Python Framework", 424 source="/Library/Frameworks/Python.framework", 425 readme="""\ 426 This package installs Python.framework, that is the python 427 interpreter and the standard library. 428 """, 429 postflight="scripts/postflight.framework", 430 selected='selected', 431 ), 432 dict( 433 name="PythonApplications", 434 long_name="GUI Applications", 435 source="/Applications/Python %(VER)s", 436 readme="""\ 437 This package installs IDLE (an interactive Python IDE), 438 Python Launcher and Build Applet (create application bundles 439 from python scripts). 440 441 It also installs a number of examples and demos. 442 """, 443 required=False, 444 selected='selected', 445 ), 446 dict( 447 name="PythonUnixTools", 448 long_name="UNIX command-line tools", 449 source="/usr/local/bin", 450 readme="""\ 451 This package installs the unix tools in /usr/local/bin for 452 compatibility with older releases of Python. This package 453 is not necessary to use Python. 454 """, 455 required=False, 456 selected='selected', 457 ), 458 dict( 459 name="PythonDocumentation", 460 long_name="Python Documentation", 461 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation", 462 source="/pydocs", 463 readme="""\ 464 This package installs the python documentation at a location 465 that is usable for pydoc and IDLE. 466 """, 467 postflight="scripts/postflight.documentation", 468 required=False, 469 selected='selected', 470 ), 471 dict( 472 name="PythonProfileChanges", 473 long_name="Shell profile updater", 474 readme="""\ 475 This packages updates your shell profile to make sure that 476 the Python tools are found by your shell in preference of 477 the system provided Python tools. 478 479 If you don't install this package you'll have to add 480 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin" 481 to your PATH by hand. 482 """, 483 postflight="scripts/postflight.patch-profile", 484 topdir="/Library/Frameworks/Python.framework", 485 source="/empty-dir", 486 required=False, 487 selected='selected', 488 ), 489 dict( 490 name="PythonInstallPip", 491 long_name="Install or upgrade pip", 492 readme="""\ 493 This package installs (or upgrades from an earlier version) 494 pip, a tool for installing and managing Python packages. 495 """, 496 postflight="scripts/postflight.ensurepip", 497 topdir="/Library/Frameworks/Python.framework", 498 source="/empty-dir", 499 required=False, 500 selected='selected', 501 ), 502 ] 503 504 return result 505 506def fatal(msg): 507 """ 508 A fatal error, bail out. 509 """ 510 sys.stderr.write('FATAL: ') 511 sys.stderr.write(msg) 512 sys.stderr.write('\n') 513 sys.exit(1) 514 515def fileContents(fn): 516 """ 517 Return the contents of the named file 518 """ 519 return open(fn, 'r').read() 520 521def runCommand(commandline): 522 """ 523 Run a command and raise RuntimeError if it fails. Output is suppressed 524 unless the command fails. 525 """ 526 fd = os.popen(commandline, 'r') 527 data = fd.read() 528 xit = fd.close() 529 if xit is not None: 530 sys.stdout.write(data) 531 raise RuntimeError("command failed: %s"%(commandline,)) 532 533 if VERBOSE: 534 sys.stdout.write(data); sys.stdout.flush() 535 536def captureCommand(commandline): 537 fd = os.popen(commandline, 'r') 538 data = fd.read() 539 xit = fd.close() 540 if xit is not None: 541 sys.stdout.write(data) 542 raise RuntimeError("command failed: %s"%(commandline,)) 543 544 return data 545 546def getTclTkVersion(configfile, versionline): 547 """ 548 search Tcl or Tk configuration file for version line 549 """ 550 try: 551 f = open(configfile, "r") 552 except OSError: 553 fatal("Framework configuration file not found: %s" % configfile) 554 555 for l in f: 556 if l.startswith(versionline): 557 f.close() 558 return l 559 560 fatal("Version variable %s not found in framework configuration file: %s" 561 % (versionline, configfile)) 562 563def checkEnvironment(): 564 """ 565 Check that we're running on a supported system. 566 """ 567 568 if sys.version_info[0:2] < (2, 7): 569 fatal("This script must be run with Python 2.7 (or later)") 570 571 if platform.system() != 'Darwin': 572 fatal("This script should be run on a macOS 10.5 (or later) system") 573 574 if int(platform.release().split('.')[0]) < 8: 575 fatal("This script should be run on a macOS 10.5 (or later) system") 576 577 # Because we only support dynamic load of only one major/minor version of 578 # Tcl/Tk, if we are not using building and using our own private copy of 579 # Tcl/Tk, ensure: 580 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked 581 # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer 582 # enforce that the version of the user-installed framework also 583 # exists in the system-supplied Tcl/Tk frameworks. Time to support 584 # Tcl/Tk 8.6 even if Apple does not. 585 if not internalTk(): 586 frameworks = {} 587 for framework in ['Tcl', 'Tk']: 588 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework 589 libfw = os.path.join('/', fwpth) 590 usrfw = os.path.join(os.getenv('HOME'), fwpth) 591 frameworks[framework] = os.readlink(libfw) 592 if not os.path.exists(libfw): 593 fatal("Please install a link to a current %s %s as %s so " 594 "the user can override the system framework." 595 % (framework, frameworks[framework], libfw)) 596 if os.path.exists(usrfw): 597 fatal("Please rename %s to avoid possible dynamic load issues." 598 % usrfw) 599 600 if frameworks['Tcl'] != frameworks['Tk']: 601 fatal("The Tcl and Tk frameworks are not the same version.") 602 603 print(" -- Building with external Tcl/Tk %s frameworks" 604 % frameworks['Tk']) 605 606 # add files to check after build 607 EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 608 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" 609 % frameworks['Tcl'], 610 "/Library/Frameworks/Tk.framework/Versions/%s/Tk" 611 % frameworks['Tk'], 612 ] 613 else: 614 print(" -- Building private copy of Tcl/Tk") 615 print("") 616 617 # Remove inherited environment variables which might influence build 618 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 619 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] 620 for ev in list(os.environ): 621 for prefix in environ_var_prefixes: 622 if ev.startswith(prefix) : 623 print("INFO: deleting environment variable %s=%s" % ( 624 ev, os.environ[ev])) 625 del os.environ[ev] 626 627 base_path = '/bin:/sbin:/usr/bin:/usr/sbin' 628 if 'SDK_TOOLS_BIN' in os.environ: 629 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path 630 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; 631 # add its fixed location here if it exists 632 OLD_DEVELOPER_TOOLS = '/Developer/Tools' 633 if os.path.isdir(OLD_DEVELOPER_TOOLS): 634 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS 635 os.environ['PATH'] = base_path 636 print("Setting default PATH: %s"%(os.environ['PATH'])) 637 638def parseOptions(args=None): 639 """ 640 Parse arguments and update global settings. 641 """ 642 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET 643 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX 644 global FW_VERSION_PREFIX 645 global FW_SSL_DIRECTORY 646 647 if args is None: 648 args = sys.argv[1:] 649 650 try: 651 options, args = getopt.getopt(args, '?hb', 652 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', 653 'dep-target=', 'universal-archs=', 'help' ]) 654 except getopt.GetoptError: 655 print(sys.exc_info()[1]) 656 sys.exit(1) 657 658 if args: 659 print("Additional arguments") 660 sys.exit(1) 661 662 deptarget = None 663 for k, v in options: 664 if k in ('-h', '-?', '--help'): 665 print(USAGE) 666 sys.exit(0) 667 668 elif k in ('-d', '--build-dir'): 669 WORKDIR=v 670 671 elif k in ('--third-party',): 672 DEPSRC=v 673 674 elif k in ('--sdk-path',): 675 print(" WARNING: --sdk-path is no longer supported") 676 677 elif k in ('--src-dir',): 678 SRCDIR=v 679 680 elif k in ('--dep-target', ): 681 DEPTARGET=v 682 deptarget=v 683 684 elif k in ('--universal-archs', ): 685 if v in UNIVERSALOPTS: 686 UNIVERSALARCHS = v 687 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 688 if deptarget is None: 689 # Select alternate default deployment 690 # target 691 DEPTARGET = default_target_map.get(v, '10.5') 692 else: 693 raise NotImplementedError(v) 694 695 else: 696 raise NotImplementedError(k) 697 698 SRCDIR=os.path.abspath(SRCDIR) 699 WORKDIR=os.path.abspath(WORKDIR) 700 DEPSRC=os.path.abspath(DEPSRC) 701 702 CC, CXX = getTargetCompilers() 703 704 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()] 705 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"] 706 707 print("-- Settings:") 708 print(" * Source directory: %s" % SRCDIR) 709 print(" * Build directory: %s" % WORKDIR) 710 print(" * Third-party source: %s" % DEPSRC) 711 print(" * Deployment target: %s" % DEPTARGET) 712 print(" * Universal archs: %s" % str(ARCHLIST)) 713 print(" * C compiler: %s" % CC) 714 print(" * C++ compiler: %s" % CXX) 715 print("") 716 print(" -- Building a Python %s framework at patch level %s" 717 % (getVersion(), getFullVersion())) 718 print("") 719 720def extractArchive(builddir, archiveName): 721 """ 722 Extract a source archive into 'builddir'. Returns the path of the 723 extracted archive. 724 725 XXX: This function assumes that archives contain a toplevel directory 726 that is has the same name as the basename of the archive. This is 727 safe enough for almost anything we use. Unfortunately, it does not 728 work for current Tcl and Tk source releases where the basename of 729 the archive ends with "-src" but the uncompressed directory does not. 730 For now, just special case Tcl and Tk tar.gz downloads. 731 """ 732 curdir = os.getcwd() 733 try: 734 os.chdir(builddir) 735 if archiveName.endswith('.tar.gz'): 736 retval = os.path.basename(archiveName[:-7]) 737 if ((retval.startswith('tcl') or retval.startswith('tk')) 738 and retval.endswith('-src')): 739 retval = retval[:-4] 740 # Strip rcxx suffix from Tcl/Tk release candidates 741 retval_rc = retval.find('rc') 742 if retval_rc > 0: 743 retval = retval[:retval_rc] 744 if os.path.exists(retval): 745 shutil.rmtree(retval) 746 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') 747 748 elif archiveName.endswith('.tar.bz2'): 749 retval = os.path.basename(archiveName[:-8]) 750 if os.path.exists(retval): 751 shutil.rmtree(retval) 752 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r') 753 754 elif archiveName.endswith('.tar'): 755 retval = os.path.basename(archiveName[:-4]) 756 if os.path.exists(retval): 757 shutil.rmtree(retval) 758 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r') 759 760 elif archiveName.endswith('.zip'): 761 retval = os.path.basename(archiveName[:-4]) 762 if os.path.exists(retval): 763 shutil.rmtree(retval) 764 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r') 765 766 data = fp.read() 767 xit = fp.close() 768 if xit is not None: 769 sys.stdout.write(data) 770 raise RuntimeError("Cannot extract %s"%(archiveName,)) 771 772 return os.path.join(builddir, retval) 773 774 finally: 775 os.chdir(curdir) 776 777def downloadURL(url, fname): 778 """ 779 Download the contents of the url into the file. 780 """ 781 fpIn = urllib_request.urlopen(url) 782 fpOut = open(fname, 'wb') 783 block = fpIn.read(10240) 784 try: 785 while block: 786 fpOut.write(block) 787 block = fpIn.read(10240) 788 fpIn.close() 789 fpOut.close() 790 except: 791 try: 792 os.unlink(fname) 793 except OSError: 794 pass 795 796def verifyThirdPartyFile(url, checksum, fname): 797 """ 798 Download file from url to filename fname if it does not already exist. 799 Abort if file contents does not match supplied md5 checksum. 800 """ 801 name = os.path.basename(fname) 802 if os.path.exists(fname): 803 print("Using local copy of %s"%(name,)) 804 else: 805 print("Did not find local copy of %s"%(name,)) 806 print("Downloading %s"%(name,)) 807 downloadURL(url, fname) 808 print("Archive for %s stored as %s"%(name, fname)) 809 if len(checksum) == 32: 810 algo = 'md5' 811 elif len(checksum) == 64: 812 algo = 'sha256' 813 else: 814 raise ValueError(checksum) 815 if os.system( 816 'CHECKSUM=$(openssl %s %s) ; test "${CHECKSUM##*= }" = "%s"' 817 % (algo, shellQuote(fname), checksum) ): 818 fatal('%s checksum mismatch for file %s' % (algo, fname)) 819 820def build_universal_openssl(basedir, archList): 821 """ 822 Special case build recipe for universal build of openssl. 823 824 The upstream OpenSSL build system does not directly support 825 OS X universal builds. We need to build each architecture 826 separately then lipo them together into fat libraries. 827 """ 828 829 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4). 830 # If we are building on a 10.4.x or earlier system, 831 # unilaterally disable assembly code building to avoid the problem. 832 no_asm = int(platform.release().split(".")[0]) < 9 833 834 def build_openssl_arch(archbase, arch): 835 "Build one architecture of openssl" 836 arch_opts = { 837 "i386": ["darwin-i386-cc"], 838 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], 839 "arm64": ["darwin64-arm64-cc"], 840 "ppc": ["darwin-ppc-cc"], 841 "ppc64": ["darwin64-ppc-cc"], 842 } 843 844 # Somewhere between OpenSSL 1.1.0j and 1.1.1c, changes cause the 845 # "enable-ec_nistp_64_gcc_128" option to get compile errors when 846 # building on our 10.6 gcc-4.2 environment. There have been other 847 # reports of projects running into this when using older compilers. 848 # So, for now, do not try to use "enable-ec_nistp_64_gcc_128" when 849 # building for 10.6. 850 if getDeptargetTuple() == (10, 6): 851 arch_opts['x86_64'].remove('enable-ec_nistp_64_gcc_128') 852 853 configure_opts = [ 854 "no-idea", 855 "no-mdc2", 856 "no-rc5", 857 "no-zlib", 858 "no-ssl3", 859 # "enable-unit-test", 860 "shared", 861 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), 862 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), 863 ] 864 if no_asm: 865 configure_opts.append("no-asm") 866 runCommand(" ".join(["perl", "Configure"] 867 + arch_opts[arch] + configure_opts)) 868 runCommand("make depend") 869 runCommand("make all") 870 runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase)) 871 # runCommand("make test") 872 return 873 874 srcdir = os.getcwd() 875 universalbase = os.path.join(srcdir, "..", 876 os.path.basename(srcdir) + "-universal") 877 os.mkdir(universalbase) 878 archbasefws = [] 879 for arch in archList: 880 # fresh copy of the source tree 881 archsrc = os.path.join(universalbase, arch, "src") 882 shutil.copytree(srcdir, archsrc, symlinks=True) 883 # install base for this arch 884 archbase = os.path.join(universalbase, arch, "root") 885 os.mkdir(archbase) 886 # Python framework base within install_prefix: 887 # the build will install into this framework.. 888 # This is to ensure that the resulting shared libs have 889 # the desired real install paths built into them. 890 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX) 891 892 # build one architecture 893 os.chdir(archsrc) 894 build_openssl_arch(archbase, arch) 895 os.chdir(srcdir) 896 archbasefws.append(archbasefw) 897 898 # copy arch-independent files from last build into the basedir framework 899 basefw = os.path.join(basedir, *FW_VERSION_PREFIX) 900 shutil.copytree( 901 os.path.join(archbasefw, "include", "openssl"), 902 os.path.join(basefw, "include", "openssl") 903 ) 904 905 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"), 906 "SHLIB_VERSION_NUMBER") 907 # e.g. -> "1.0.0" 908 libcrypto = "libcrypto.dylib" 909 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".") 910 # e.g. -> "libcrypto.1.0.0.dylib" 911 libssl = "libssl.dylib" 912 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".") 913 # e.g. -> "libssl.1.0.0.dylib" 914 915 try: 916 os.mkdir(os.path.join(basefw, "lib")) 917 except OSError: 918 pass 919 920 # merge the individual arch-dependent shared libs into a fat shared lib 921 archbasefws.insert(0, basefw) 922 for (lib_unversioned, lib_versioned) in [ 923 (libcrypto, libcrypto_versioned), 924 (libssl, libssl_versioned) 925 ]: 926 runCommand("lipo -create -output " + 927 " ".join(shellQuote( 928 os.path.join(fw, "lib", lib_versioned)) 929 for fw in archbasefws)) 930 # and create an unversioned symlink of it 931 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned)) 932 933 # Create links in the temp include and lib dirs that will be injected 934 # into the Python build so that setup.py can find them while building 935 # and the versioned links so that the setup.py post-build import test 936 # does not fail. 937 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX) 938 for fn in [ 939 ["include", "openssl"], 940 ["lib", libcrypto], 941 ["lib", libssl], 942 ["lib", libcrypto_versioned], 943 ["lib", libssl_versioned], 944 ]: 945 os.symlink( 946 os.path.join(relative_path, *fn), 947 os.path.join(basedir, "usr", "local", *fn) 948 ) 949 950 return 951 952def buildRecipe(recipe, basedir, archList): 953 """ 954 Build software using a recipe. This function does the 955 'configure;make;make install' dance for C software, with a possibility 956 to customize this process, basically a poor-mans DarwinPorts. 957 """ 958 curdir = os.getcwd() 959 960 name = recipe['name'] 961 THIRD_PARTY_LIBS.append(name) 962 url = recipe['url'] 963 configure = recipe.get('configure', './configure') 964 buildrecipe = recipe.get('buildrecipe', None) 965 install = recipe.get('install', 'make && make install DESTDIR=%s'%( 966 shellQuote(basedir))) 967 968 archiveName = os.path.split(url)[-1] 969 sourceArchive = os.path.join(DEPSRC, archiveName) 970 971 if not os.path.exists(DEPSRC): 972 os.mkdir(DEPSRC) 973 974 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive) 975 print("Extracting archive for %s"%(name,)) 976 buildDir=os.path.join(WORKDIR, '_bld') 977 if not os.path.exists(buildDir): 978 os.mkdir(buildDir) 979 980 workDir = extractArchive(buildDir, sourceArchive) 981 os.chdir(workDir) 982 983 for patch in recipe.get('patches', ()): 984 if isinstance(patch, tuple): 985 url, checksum = patch 986 fn = os.path.join(DEPSRC, os.path.basename(url)) 987 verifyThirdPartyFile(url, checksum, fn) 988 else: 989 # patch is a file in the source directory 990 fn = os.path.join(curdir, patch) 991 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1), 992 shellQuote(fn),)) 993 994 for patchscript in recipe.get('patchscripts', ()): 995 if isinstance(patchscript, tuple): 996 url, checksum = patchscript 997 fn = os.path.join(DEPSRC, os.path.basename(url)) 998 verifyThirdPartyFile(url, checksum, fn) 999 else: 1000 # patch is a file in the source directory 1001 fn = os.path.join(curdir, patchscript) 1002 if fn.endswith('.bz2'): 1003 runCommand('bunzip2 -fk %s' % shellQuote(fn)) 1004 fn = fn[:-4] 1005 runCommand('sh %s' % shellQuote(fn)) 1006 os.unlink(fn) 1007 1008 if 'buildDir' in recipe: 1009 os.chdir(recipe['buildDir']) 1010 1011 if configure is not None: 1012 configure_args = [ 1013 "--prefix=/usr/local", 1014 "--enable-static", 1015 "--disable-shared", 1016 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),), 1017 ] 1018 1019 if 'configure_pre' in recipe: 1020 args = list(recipe['configure_pre']) 1021 if '--disable-static' in args: 1022 configure_args.remove('--enable-static') 1023 if '--enable-shared' in args: 1024 configure_args.remove('--disable-shared') 1025 configure_args.extend(args) 1026 1027 if recipe.get('useLDFlags', 1): 1028 configure_args.extend([ 1029 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1030 "-I%s/usr/local/include"%( 1031 recipe.get('extra_cflags', ''), 1032 DEPTARGET, 1033 ' -arch '.join(archList), 1034 shellQuote(basedir)[1:-1],), 1035 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( 1036 DEPTARGET, 1037 shellQuote(basedir)[1:-1], 1038 ' -arch '.join(archList)), 1039 ]) 1040 else: 1041 configure_args.extend([ 1042 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1043 "-I%s/usr/local/include"%( 1044 recipe.get('extra_cflags', ''), 1045 DEPTARGET, 1046 ' -arch '.join(archList), 1047 shellQuote(basedir)[1:-1],), 1048 ]) 1049 1050 if 'configure_post' in recipe: 1051 configure_args = configure_args + list(recipe['configure_post']) 1052 1053 configure_args.insert(0, configure) 1054 configure_args = [ shellQuote(a) for a in configure_args ] 1055 1056 print("Running configure for %s"%(name,)) 1057 runCommand(' '.join(configure_args) + ' 2>&1') 1058 1059 if buildrecipe is not None: 1060 # call special-case build recipe, e.g. for openssl 1061 buildrecipe(basedir, archList) 1062 1063 if install is not None: 1064 print("Running install for %s"%(name,)) 1065 runCommand('{ ' + install + ' ;} 2>&1') 1066 1067 print("Done %s"%(name,)) 1068 print("") 1069 1070 os.chdir(curdir) 1071 1072def buildLibraries(): 1073 """ 1074 Build our dependencies into $WORKDIR/libraries/usr/local 1075 """ 1076 print("") 1077 print("Building required libraries") 1078 print("") 1079 universal = os.path.join(WORKDIR, 'libraries') 1080 os.mkdir(universal) 1081 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib')) 1082 os.makedirs(os.path.join(universal, 'usr', 'local', 'include')) 1083 1084 for recipe in library_recipes(): 1085 buildRecipe(recipe, universal, ARCHLIST) 1086 1087 1088 1089def buildPythonDocs(): 1090 # This stores the documentation as Resources/English.lproj/Documentation 1091 # inside the framework. pydoc and IDLE will pick it up there. 1092 print("Install python documentation") 1093 rootDir = os.path.join(WORKDIR, '_root') 1094 buildDir = os.path.join('../../Doc') 1095 docdir = os.path.join(rootDir, 'pydocs') 1096 curDir = os.getcwd() 1097 os.chdir(buildDir) 1098 runCommand('make clean') 1099 1100 # Search third-party source directory for a pre-built version of the docs. 1101 # Use the naming convention of the docs.python.org html downloads: 1102 # python-3.9.0b1-docs-html.tar.bz2 1103 doctarfiles = [ f for f in os.listdir(DEPSRC) 1104 if f.startswith('python-'+getFullVersion()) 1105 if f.endswith('-docs-html.tar.bz2') ] 1106 if doctarfiles: 1107 doctarfile = doctarfiles[0] 1108 if not os.path.exists('build'): 1109 os.mkdir('build') 1110 # if build directory existed, it was emptied by make clean, above 1111 os.chdir('build') 1112 # Extract the first archive found for this version into build 1113 runCommand('tar xjf %s'%shellQuote(os.path.join(DEPSRC, doctarfile))) 1114 # see if tar extracted a directory ending in -docs-html 1115 archivefiles = [ f for f in os.listdir('.') 1116 if f.endswith('-docs-html') 1117 if os.path.isdir(f) ] 1118 if archivefiles: 1119 archivefile = archivefiles[0] 1120 # make it our 'Docs/build/html' directory 1121 print(' -- using pre-built python documentation from %s'%archivefile) 1122 os.rename(archivefile, 'html') 1123 os.chdir(buildDir) 1124 1125 htmlDir = os.path.join('build', 'html') 1126 if not os.path.exists(htmlDir): 1127 # Create virtual environment for docs builds with blurb and sphinx 1128 runCommand('make venv') 1129 runCommand('make html PYTHON=venv/bin/python') 1130 os.rename(htmlDir, docdir) 1131 os.chdir(curDir) 1132 1133 1134def buildPython(): 1135 print("Building a universal python for %s architectures" % UNIVERSALARCHS) 1136 1137 buildDir = os.path.join(WORKDIR, '_bld', 'python') 1138 rootDir = os.path.join(WORKDIR, '_root') 1139 1140 if os.path.exists(buildDir): 1141 shutil.rmtree(buildDir) 1142 if os.path.exists(rootDir): 1143 shutil.rmtree(rootDir) 1144 os.makedirs(buildDir) 1145 os.makedirs(rootDir) 1146 os.makedirs(os.path.join(rootDir, 'empty-dir')) 1147 curdir = os.getcwd() 1148 os.chdir(buildDir) 1149 1150 # Extract the version from the configure file, needed to calculate 1151 # several paths. 1152 version = getVersion() 1153 1154 # Since the extra libs are not in their installed framework location 1155 # during the build, augment the library path so that the interpreter 1156 # will find them during its extension import sanity checks. 1157 1158 print("Running configure...") 1159 print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") 1160 runCommand("%s -C --enable-framework --enable-universalsdk=/ " 1161 "--with-mimalloc=no " 1162 "--with-system-libmpdec " 1163 "--with-universal-archs=%s " 1164 "%s " 1165 "%s " 1166 "%s " 1167 "%s " 1168 "%s " 1169 "%s " 1170 "LDFLAGS='-g -L%s/libraries/usr/local/lib' " 1171 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( 1172 shellQuote(os.path.join(SRCDIR, 'configure')), 1173 UNIVERSALARCHS, 1174 (' ', '--with-computed-gotos ')[PYTHON_3], 1175 (' ', '--without-ensurepip ')[PYTHON_3], 1176 (' ', "--with-openssl='%s/libraries/usr/local'"%( 1177 shellQuote(WORKDIR)[1:-1],))[PYTHON_3], 1178 (' ', "--enable-optimizations --with-lto")[compilerCanOptimize()], 1179 (' ', "TCLTK_CFLAGS='-I%s/libraries/usr/local/include'"%( 1180 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1181 (' ', "TCLTK_LIBS='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( 1182 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1183 shellQuote(WORKDIR)[1:-1], 1184 shellQuote(WORKDIR)[1:-1])) 1185 1186 # As of macOS 10.11 with SYSTEM INTEGRITY PROTECTION, DYLD_* 1187 # environment variables are no longer automatically inherited 1188 # by child processes from their parents. We used to just set 1189 # DYLD_LIBRARY_PATH, pointing to the third-party libs, 1190 # in build-installer.py's process environment and it was 1191 # passed through the make utility into the environment of 1192 # setup.py. Instead, we now append DYLD_LIBRARY_PATH to 1193 # the existing RUNSHARED configuration value when we call 1194 # make for extension module builds. 1195 1196 runshared_for_make = "".join([ 1197 " RUNSHARED=", 1198 "'", 1199 grepValue("Makefile", "RUNSHARED"), 1200 ' DYLD_LIBRARY_PATH=', 1201 os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib'), 1202 "'" ]) 1203 1204 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS 1205 # and, if defined, append its value to the make command. This allows 1206 # us to pass in version control tags, like GITTAG, to a build from a 1207 # tarball rather than from a vcs checkout, thus eliminating the need 1208 # to have a working copy of the vcs program on the build machine. 1209 # 1210 # A typical use might be: 1211 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ 1212 # GITVERSION='echo 123456789a' \ 1213 # GITTAG='echo v3.6.0' \ 1214 # GITBRANCH='echo 3.6'" 1215 1216 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") 1217 if make_extras: 1218 make_cmd = "make " + make_extras + runshared_for_make 1219 else: 1220 make_cmd = "make" + runshared_for_make 1221 print("Running " + make_cmd) 1222 runCommand(make_cmd) 1223 1224 make_cmd = "make install DESTDIR=%s %s"%( 1225 shellQuote(rootDir), 1226 runshared_for_make) 1227 print("Running " + make_cmd) 1228 runCommand(make_cmd) 1229 1230 make_cmd = "make frameworkinstallextras DESTDIR=%s %s"%( 1231 shellQuote(rootDir), 1232 runshared_for_make) 1233 print("Running " + make_cmd) 1234 runCommand(make_cmd) 1235 1236 print("Copying required shared libraries") 1237 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): 1238 build_lib_dir = os.path.join( 1239 WORKDIR, 'libraries', 'Library', 'Frameworks', 1240 'Python.framework', 'Versions', getVersion(), 'lib') 1241 fw_lib_dir = os.path.join( 1242 WORKDIR, '_root', 'Library', 'Frameworks', 1243 'Python.framework', 'Versions', getVersion(), 'lib') 1244 if internalTk(): 1245 # move Tcl and Tk pkgconfig files 1246 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( 1247 shellQuote(build_lib_dir), 1248 shellQuote(fw_lib_dir) )) 1249 runCommand("rm -r %s/pkgconfig"%( 1250 shellQuote(build_lib_dir), )) 1251 runCommand("mv %s/* %s"%( 1252 shellQuote(build_lib_dir), 1253 shellQuote(fw_lib_dir) )) 1254 1255 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') 1256 frmDirVersioned = os.path.join(frmDir, 'Versions', version) 1257 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,)) 1258 # create directory for OpenSSL certificates 1259 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl') 1260 os.makedirs(sslDir) 1261 1262 print("Fix file modes") 1263 gid = grp.getgrnam('admin').gr_gid 1264 1265 shared_lib_error = False 1266 for dirpath, dirnames, filenames in os.walk(frmDir): 1267 for dn in dirnames: 1268 os.chmod(os.path.join(dirpath, dn), STAT_0o775) 1269 os.chown(os.path.join(dirpath, dn), -1, gid) 1270 1271 for fn in filenames: 1272 if os.path.islink(fn): 1273 continue 1274 1275 # "chmod g+w $fn" 1276 p = os.path.join(dirpath, fn) 1277 st = os.stat(p) 1278 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP) 1279 os.chown(p, -1, gid) 1280 1281 if fn in EXPECTED_SHARED_LIBS: 1282 # check to see that this file was linked with the 1283 # expected library path and version 1284 data = captureCommand("otool -L %s" % shellQuote(p)) 1285 for sl in EXPECTED_SHARED_LIBS[fn]: 1286 if ("\t%s " % sl) not in data: 1287 print("Expected shared lib %s was not linked with %s" 1288 % (sl, p)) 1289 shared_lib_error = True 1290 1291 if shared_lib_error: 1292 fatal("Unexpected shared library errors.") 1293 1294 if PYTHON_3: 1295 LDVERSION=None 1296 VERSION=None 1297 ABIFLAGS=None 1298 1299 fp = open(os.path.join(buildDir, 'Makefile'), 'r') 1300 for ln in fp: 1301 if ln.startswith('VERSION='): 1302 VERSION=ln.split()[1] 1303 if ln.startswith('ABIFLAGS='): 1304 ABIFLAGS=ln.split() 1305 ABIFLAGS=ABIFLAGS[1] if len(ABIFLAGS) > 1 else '' 1306 if ln.startswith('LDVERSION='): 1307 LDVERSION=ln.split()[1] 1308 fp.close() 1309 1310 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION) 1311 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS) 1312 config_suffix = '-' + LDVERSION 1313 if getVersionMajorMinor() >= (3, 6): 1314 config_suffix = config_suffix + '-darwin' 1315 else: 1316 config_suffix = '' # Python 2.x 1317 1318 # We added some directories to the search path during the configure 1319 # phase. Remove those because those directories won't be there on 1320 # the end-users system. Also remove the directories from _sysconfigdata.py 1321 # (added in 3.3) if it exists. 1322 1323 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,) 1324 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,) 1325 1326 # fix Makefile 1327 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile') 1328 fp = open(path, 'r') 1329 data = fp.read() 1330 fp.close() 1331 1332 for p in (include_path, lib_path): 1333 data = data.replace(" " + p, '') 1334 data = data.replace(p + " ", '') 1335 1336 fp = open(path, 'w') 1337 fp.write(data) 1338 fp.close() 1339 1340 # fix _sysconfigdata 1341 # 1342 # TODO: make this more robust! test_sysconfig_module of 1343 # distutils.tests.test_sysconfig.SysconfigTestCase tests that 1344 # the output from get_config_var in both sysconfig and 1345 # distutils.sysconfig is exactly the same for both CFLAGS and 1346 # LDFLAGS. The fixing up is now complicated by the pretty 1347 # printing in _sysconfigdata.py. Also, we are using the 1348 # pprint from the Python running the installer build which 1349 # may not cosmetically format the same as the pprint in the Python 1350 # being built (and which is used to originally generate 1351 # _sysconfigdata.py). 1352 1353 import pprint 1354 if getVersionMajorMinor() >= (3, 6): 1355 # XXX this is extra-fragile 1356 path = os.path.join(path_to_lib, 1357 '_sysconfigdata_%s_darwin_darwin.py' % (ABIFLAGS,)) 1358 else: 1359 path = os.path.join(path_to_lib, '_sysconfigdata.py') 1360 fp = open(path, 'r') 1361 data = fp.read() 1362 fp.close() 1363 # create build_time_vars dict 1364 if RUNNING_ON_PYTHON2: 1365 exec(data) 1366 else: 1367 g_dict = {} 1368 l_dict = {} 1369 exec(data, g_dict, l_dict) 1370 build_time_vars = l_dict['build_time_vars'] 1371 vars = {} 1372 for k, v in build_time_vars.items(): 1373 if isinstance(v, str): 1374 for p in (include_path, lib_path): 1375 v = v.replace(' ' + p, '') 1376 v = v.replace(p + ' ', '') 1377 vars[k] = v 1378 1379 fp = open(path, 'w') 1380 # duplicated from sysconfig._generate_posix_vars() 1381 fp.write('# system configuration generated and used by' 1382 ' the sysconfig module\n') 1383 fp.write('build_time_vars = ') 1384 pprint.pprint(vars, stream=fp) 1385 fp.close() 1386 1387 # Add symlinks in /usr/local/bin, using relative links 1388 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin') 1389 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks', 1390 'Python.framework', 'Versions', version, 'bin') 1391 if os.path.exists(usr_local_bin): 1392 shutil.rmtree(usr_local_bin) 1393 os.makedirs(usr_local_bin) 1394 for fn in os.listdir( 1395 os.path.join(frmDir, 'Versions', version, 'bin')): 1396 os.symlink(os.path.join(to_framework, fn), 1397 os.path.join(usr_local_bin, fn)) 1398 1399 os.chdir(curdir) 1400 1401def patchFile(inPath, outPath): 1402 data = fileContents(inPath) 1403 data = data.replace('$FULL_VERSION', getFullVersion()) 1404 data = data.replace('$VERSION', getVersion()) 1405 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later'))) 1406 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS])) 1407 data = data.replace('$INSTALL_SIZE', installSize()) 1408 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS)) 1409 1410 # This one is not handy as a template variable 1411 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework') 1412 fp = open(outPath, 'w') 1413 fp.write(data) 1414 fp.close() 1415 1416def patchScript(inPath, outPath): 1417 major, minor = getVersionMajorMinor() 1418 data = fileContents(inPath) 1419 data = data.replace('@PYMAJOR@', str(major)) 1420 data = data.replace('@PYVER@', getVersion()) 1421 fp = open(outPath, 'w') 1422 fp.write(data) 1423 fp.close() 1424 os.chmod(outPath, STAT_0o755) 1425 1426 1427 1428def packageFromRecipe(targetDir, recipe): 1429 curdir = os.getcwd() 1430 try: 1431 # The major version (such as 2.5) is included in the package name 1432 # because having two version of python installed at the same time is 1433 # common. 1434 pkgname = '%s-%s'%(recipe['name'], getVersion()) 1435 srcdir = recipe.get('source') 1436 pkgroot = recipe.get('topdir', srcdir) 1437 postflight = recipe.get('postflight') 1438 readme = textwrap.dedent(recipe['readme']) 1439 isRequired = recipe.get('required', True) 1440 1441 print("- building package %s"%(pkgname,)) 1442 1443 # Substitute some variables 1444 textvars = dict( 1445 VER=getVersion(), 1446 FULLVER=getFullVersion(), 1447 ) 1448 readme = readme % textvars 1449 1450 if pkgroot is not None: 1451 pkgroot = pkgroot % textvars 1452 else: 1453 pkgroot = '/' 1454 1455 if srcdir is not None: 1456 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:]) 1457 srcdir = srcdir % textvars 1458 1459 if postflight is not None: 1460 postflight = os.path.abspath(postflight) 1461 1462 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents') 1463 os.makedirs(packageContents) 1464 1465 if srcdir is not None: 1466 os.chdir(srcdir) 1467 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1468 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1469 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),)) 1470 1471 fn = os.path.join(packageContents, 'PkgInfo') 1472 fp = open(fn, 'w') 1473 fp.write('pmkrpkg1') 1474 fp.close() 1475 1476 rsrcDir = os.path.join(packageContents, "Resources") 1477 os.mkdir(rsrcDir) 1478 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w') 1479 fp.write(readme) 1480 fp.close() 1481 1482 if postflight is not None: 1483 patchScript(postflight, os.path.join(rsrcDir, 'postflight')) 1484 1485 vers = getFullVersion() 1486 major, minor = getVersionMajorMinor() 1487 pl = dict( 1488 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), 1489 CFBundleIdentifier='org.python.Python.%s'%(pkgname,), 1490 CFBundleName='Python.%s'%(pkgname,), 1491 CFBundleShortVersionString=vers, 1492 IFMajorVersion=major, 1493 IFMinorVersion=minor, 1494 IFPkgFormatVersion=0.10000000149011612, 1495 IFPkgFlagAllowBackRev=False, 1496 IFPkgFlagAuthorizationAction="RootAuthorization", 1497 IFPkgFlagDefaultLocation=pkgroot, 1498 IFPkgFlagFollowLinks=True, 1499 IFPkgFlagInstallFat=True, 1500 IFPkgFlagIsRequired=isRequired, 1501 IFPkgFlagOverwritePermissions=False, 1502 IFPkgFlagRelocatable=False, 1503 IFPkgFlagRestartAction="NoRestart", 1504 IFPkgFlagRootVolumeOnly=True, 1505 IFPkgFlagUpdateInstalledLanguages=False, 1506 ) 1507 writePlist(pl, os.path.join(packageContents, 'Info.plist')) 1508 1509 pl = dict( 1510 IFPkgDescriptionDescription=readme, 1511 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), 1512 IFPkgDescriptionVersion=vers, 1513 ) 1514 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist')) 1515 1516 finally: 1517 os.chdir(curdir) 1518 1519 1520def makeMpkgPlist(path): 1521 1522 vers = getFullVersion() 1523 major, minor = getVersionMajorMinor() 1524 1525 pl = dict( 1526 CFBundleGetInfoString="Python %s"%(vers,), 1527 CFBundleIdentifier='org.python.Python', 1528 CFBundleName='Python', 1529 CFBundleShortVersionString=vers, 1530 IFMajorVersion=major, 1531 IFMinorVersion=minor, 1532 IFPkgFlagComponentDirectory="Contents/Packages", 1533 IFPkgFlagPackageList=[ 1534 dict( 1535 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()), 1536 IFPkgFlagPackageSelection=item.get('selected', 'selected'), 1537 ) 1538 for item in pkg_recipes() 1539 ], 1540 IFPkgFormatVersion=0.10000000149011612, 1541 IFPkgFlagBackgroundScaling="proportional", 1542 IFPkgFlagBackgroundAlignment="left", 1543 IFPkgFlagAuthorizationAction="RootAuthorization", 1544 ) 1545 1546 writePlist(pl, path) 1547 1548 1549def buildInstaller(): 1550 1551 # Zap all compiled files 1552 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')): 1553 for fn in filenames: 1554 if fn.endswith('.pyc') or fn.endswith('.pyo'): 1555 os.unlink(os.path.join(dirpath, fn)) 1556 1557 outdir = os.path.join(WORKDIR, 'installer') 1558 if os.path.exists(outdir): 1559 shutil.rmtree(outdir) 1560 os.mkdir(outdir) 1561 1562 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents') 1563 pkgcontents = os.path.join(pkgroot, 'Packages') 1564 os.makedirs(pkgcontents) 1565 for recipe in pkg_recipes(): 1566 packageFromRecipe(pkgcontents, recipe) 1567 1568 rsrcDir = os.path.join(pkgroot, 'Resources') 1569 1570 fn = os.path.join(pkgroot, 'PkgInfo') 1571 fp = open(fn, 'w') 1572 fp.write('pmkrpkg1') 1573 fp.close() 1574 1575 os.mkdir(rsrcDir) 1576 1577 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) 1578 pl = dict( 1579 IFPkgDescriptionTitle="Python", 1580 IFPkgDescriptionVersion=getVersion(), 1581 ) 1582 1583 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist')) 1584 for fn in os.listdir('resources'): 1585 if fn == '.svn': continue 1586 if fn.endswith('.jpg'): 1587 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1588 else: 1589 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1590 1591 1592def installSize(clear=False, _saved=[]): 1593 if clear: 1594 del _saved[:] 1595 if not _saved: 1596 data = captureCommand("du -ks %s"%( 1597 shellQuote(os.path.join(WORKDIR, '_root')))) 1598 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),)) 1599 return _saved[0] 1600 1601 1602def buildDMG(): 1603 """ 1604 Create DMG containing the rootDir. 1605 """ 1606 outdir = os.path.join(WORKDIR, 'diskimage') 1607 if os.path.exists(outdir): 1608 shutil.rmtree(outdir) 1609 1610 # We used to use the deployment target as the last characters of the 1611 # installer file name. With the introduction of weaklinked installer 1612 # variants, we may have two variants with the same file name, i.e. 1613 # both ending in '10.9'. To avoid this, we now use the major/minor 1614 # version numbers of the macOS version we are building on. 1615 # Also, as of macOS 11, operating system version numbering has 1616 # changed from three components to two, i.e. 1617 # 10.14.1, 10.14.2, ... 1618 # 10.15.1, 10.15.2, ... 1619 # 11.1, 11.2, ... 1620 # 12.1, 12.2, ... 1621 # (A further twist is that, when running on macOS 11, binaries built 1622 # on older systems may be shown an operating system version of 10.16 1623 # instead of 11. We should not run into that situation here.) 1624 # Also we should use "macos" instead of "macosx" going forward. 1625 # 1626 # To maintain compatibility for legacy variants, the file name for 1627 # builds on macOS 10.15 and earlier remains: 1628 # python-3.x.y-macosx10.z.{dmg->pkg} 1629 # e.g. python-3.9.4-macosx10.9.{dmg->pkg} 1630 # and for builds on macOS 11+: 1631 # python-3.x.y-macosz.{dmg->pkg} 1632 # e.g. python-3.9.4-macos11.{dmg->pkg} 1633 1634 build_tuple = getBuildTuple() 1635 if build_tuple[0] < 11: 1636 os_name = 'macosx' 1637 build_system_version = '%s.%s' % build_tuple 1638 else: 1639 os_name = 'macos' 1640 build_system_version = str(build_tuple[0]) 1641 imagepath = os.path.join(outdir, 1642 'python-%s-%s%s'%(getFullVersion(),os_name,build_system_version)) 1643 if INCLUDE_TIMESTAMP: 1644 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3]) 1645 imagepath = imagepath + '.dmg' 1646 1647 os.mkdir(outdir) 1648 1649 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9, 1650 # when hdiutil create fails with "Resource busy". For now, just retry 1651 # the create a few times and hope that it eventually works. 1652 1653 volname='Python %s'%(getFullVersion()) 1654 cmd = ("hdiutil create -format UDRW -volname %s -srcfolder %s -size 100m %s"%( 1655 shellQuote(volname), 1656 shellQuote(os.path.join(WORKDIR, 'installer')), 1657 shellQuote(imagepath + ".tmp.dmg" ))) 1658 for i in range(5): 1659 fd = os.popen(cmd, 'r') 1660 data = fd.read() 1661 xit = fd.close() 1662 if not xit: 1663 break 1664 sys.stdout.write(data) 1665 print(" -- retrying hdiutil create") 1666 time.sleep(5) 1667 else: 1668 raise RuntimeError("command failed: %s"%(cmd,)) 1669 1670 if not os.path.exists(os.path.join(WORKDIR, "mnt")): 1671 os.mkdir(os.path.join(WORKDIR, "mnt")) 1672 runCommand("hdiutil attach %s -mountroot %s"%( 1673 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt")))) 1674 1675 # Custom icon for the DMG, shown when the DMG is mounted. 1676 shutil.copy("../Icons/Disk Image.icns", 1677 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns")) 1678 runCommand("SetFile -a C %s/"%( 1679 shellQuote(os.path.join(WORKDIR, "mnt", volname)),)) 1680 1681 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname)))) 1682 1683 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns") 1684 runCommand("hdiutil convert %s -format UDZO -o %s"%( 1685 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath))) 1686 setIcon(imagepath, "../Icons/Disk Image.icns") 1687 1688 os.unlink(imagepath + ".tmp.dmg") 1689 1690 return imagepath 1691 1692 1693def setIcon(filePath, icnsPath): 1694 """ 1695 Set the custom icon for the specified file or directory. 1696 """ 1697 1698 dirPath = os.path.normpath(os.path.dirname(__file__)) 1699 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon") 1700 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime: 1701 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due 1702 # to connections to the window server. 1703 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS") 1704 if not os.path.exists(appPath): 1705 os.makedirs(appPath) 1706 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%( 1707 shellQuote(toolPath), shellQuote(dirPath))) 1708 1709 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath), 1710 shellQuote(filePath))) 1711 1712def main(): 1713 # First parse options and check if we can perform our work 1714 parseOptions() 1715 checkEnvironment() 1716 1717 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET 1718 os.environ['CC'] = CC 1719 os.environ['CXX'] = CXX 1720 1721 if os.path.exists(WORKDIR): 1722 shutil.rmtree(WORKDIR) 1723 os.mkdir(WORKDIR) 1724 1725 os.environ['LC_ALL'] = 'C' 1726 1727 # Then build third-party libraries such as sleepycat DB4. 1728 buildLibraries() 1729 1730 # Now build python itself 1731 buildPython() 1732 1733 # And then build the documentation 1734 # Remove the Deployment Target from the shell 1735 # environment, it's no longer needed and 1736 # an unexpected build target can cause problems 1737 # when Sphinx and its dependencies need to 1738 # be (re-)installed. 1739 del os.environ['MACOSX_DEPLOYMENT_TARGET'] 1740 buildPythonDocs() 1741 1742 1743 # Prepare the applications folder 1744 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%( 1745 getVersion(),)) 1746 fn = os.path.join(folder, "License.rtf") 1747 patchFile("resources/License.rtf", fn) 1748 fn = os.path.join(folder, "ReadMe.rtf") 1749 patchFile("resources/ReadMe.rtf", fn) 1750 fn = os.path.join(folder, "Update Shell Profile.command") 1751 patchScript("scripts/postflight.patch-profile", fn) 1752 fn = os.path.join(folder, "Install Certificates.command") 1753 patchScript("resources/install_certificates.command", fn) 1754 os.chmod(folder, STAT_0o755) 1755 setIcon(folder, "../Icons/Python Folder.icns") 1756 1757 # Create the installer 1758 buildInstaller() 1759 1760 # And copy the readme into the directory containing the installer 1761 patchFile('resources/ReadMe.rtf', 1762 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf')) 1763 1764 # Ditto for the license file. 1765 patchFile('resources/License.rtf', 1766 os.path.join(WORKDIR, 'installer', 'License.rtf')) 1767 1768 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w') 1769 fp.write("# BUILD INFO\n") 1770 fp.write("# Date: %s\n" % time.ctime()) 1771 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos) 1772 fp.close() 1773 1774 # And copy it to a DMG 1775 buildDMG() 1776 1777if __name__ == "__main__": 1778 main() 1779