• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2"""
3This script is used to build "official" universal installers on Mac OS X.
4It requires at least Mac OS X 10.5, Xcode 3, and the 10.4u SDK for
532-bit builds.  64-bit or four-way universal builds require at least
6OS X 10.5 and the 10.5 SDK.
7
8Please ensure that this script keeps working with Python 2.5, to avoid
9bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5).  Sphinx,
10which is used to build the documentation, currently requires at least
11Python 2.4.  However, as of Python 3.4.1, Doc builds require an external
12sphinx-build and the current versions of Sphinx now require at least
13Python 2.6.
14
15In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script
16requires an installed version of hg and a third-party version of
17Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5
18(for 10.6 or later) installed in /Library/Frameworks.  When installed,
19the Python built by this script will attempt to dynamically link first to
20Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall
21back to the ones in /System/Library/Framework.  For the build, we recommend
22installing the most recent ActiveTcl 8.4 or 8.5 version.
23
2432-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5
25and the installation of additional components, such as a newer Python
26(2.5 is needed for Python parser updates), hg, and for the documentation
27build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later).
28
29Usage: see USAGE variable in the script.
30"""
31import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp
32try:
33    import urllib2 as urllib_request
34except ImportError:
35    import urllib.request as urllib_request
36
37STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
38             | stat.S_IRGRP |                stat.S_IXGRP
39             | stat.S_IROTH |                stat.S_IXOTH )
40
41STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
42             | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP
43             | stat.S_IROTH |                stat.S_IXOTH )
44
45INCLUDE_TIMESTAMP = 1
46VERBOSE = 1
47
48from plistlib import Plist
49
50try:
51    from plistlib import writePlist
52except ImportError:
53    # We're run using python2.3
54    def writePlist(plist, path):
55        plist.write(path)
56
57def shellQuote(value):
58    """
59    Return the string value in a form that can safely be inserted into
60    a shell command.
61    """
62    return "'%s'"%(value.replace("'", "'\"'\"'"))
63
64def grepValue(fn, variable):
65    """
66    Return the unquoted value of a variable from a file..
67    QUOTED_VALUE='quotes'    -> str('quotes')
68    UNQUOTED_VALUE=noquotes  -> str('noquotes')
69    """
70    variable = variable + '='
71    for ln in open(fn, 'r'):
72        if ln.startswith(variable):
73            value = ln[len(variable):].strip()
74            return value.strip("\"'")
75    raise RuntimeError("Cannot find variable %s" % variable[:-1])
76
77_cache_getVersion = None
78
79def getVersion():
80    global _cache_getVersion
81    if _cache_getVersion is None:
82        _cache_getVersion = grepValue(
83            os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION')
84    return _cache_getVersion
85
86def getVersionMajorMinor():
87    return tuple([int(n) for n in getVersion().split('.', 2)])
88
89_cache_getFullVersion = None
90
91def getFullVersion():
92    global _cache_getFullVersion
93    if _cache_getFullVersion is not None:
94        return _cache_getFullVersion
95    fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h')
96    for ln in open(fn):
97        if 'PY_VERSION' in ln:
98            _cache_getFullVersion = ln.split()[-1][1:-1]
99            return _cache_getFullVersion
100    raise RuntimeError("Cannot find full version??")
101
102FW_PREFIX = ["Library", "Frameworks", "Python.framework"]
103FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions
104
105# The directory we'll use to create the build (will be erased and recreated)
106WORKDIR = "/tmp/_py"
107
108# The directory we'll use to store third-party sources. Set this to something
109# else if you don't want to re-fetch required libraries every time.
110DEPSRC = os.path.join(WORKDIR, 'third-party')
111DEPSRC = os.path.expanduser('~/Universal/other-sources')
112
113# Location of the preferred SDK
114
115### There are some issues with the SDK selection below here,
116### The resulting binary doesn't work on all platforms that
117### it should. Always default to the 10.4u SDK until that
118### issue is resolved.
119###
120##if int(os.uname()[2].split('.')[0]) == 8:
121##    # Explicitly use the 10.4u (universal) SDK when
122##    # building on 10.4, the system headers are not
123##    # useable for a universal build
124##    SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk"
125##else:
126##    SDKPATH = "/"
127
128SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk"
129
130universal_opts_map = { '32-bit': ('i386', 'ppc',),
131                       '64-bit': ('x86_64', 'ppc64',),
132                       'intel':  ('i386', 'x86_64'),
133                       '3-way':  ('ppc', 'i386', 'x86_64'),
134                       'all':    ('i386', 'ppc', 'x86_64', 'ppc64',) }
135default_target_map = {
136        '64-bit': '10.5',
137        '3-way': '10.5',
138        'intel': '10.5',
139        'all': '10.5',
140}
141
142UNIVERSALOPTS = tuple(universal_opts_map.keys())
143
144UNIVERSALARCHS = '32-bit'
145
146ARCHLIST = universal_opts_map[UNIVERSALARCHS]
147
148# Source directory (assume we're in Mac/BuildScript)
149SRCDIR = os.path.dirname(
150        os.path.dirname(
151            os.path.dirname(
152                os.path.abspath(__file__
153        ))))
154
155# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level
156DEPTARGET = '10.3'
157
158def getDeptargetTuple():
159    return tuple([int(n) for n in DEPTARGET.split('.')[0:2]])
160
161def getTargetCompilers():
162    target_cc_map = {
163        '10.3': ('gcc-4.0', 'g++-4.0'),
164        '10.4': ('gcc-4.0', 'g++-4.0'),
165        '10.5': ('gcc-4.2', 'g++-4.2'),
166        '10.6': ('gcc-4.2', 'g++-4.2'),
167    }
168    return target_cc_map.get(DEPTARGET, ('clang', 'clang++') )
169
170CC, CXX = getTargetCompilers()
171
172PYTHON_3 = getVersionMajorMinor() >= (3, 0)
173
174USAGE = textwrap.dedent("""\
175    Usage: build_python [options]
176
177    Options:
178    -? or -h:            Show this message
179    -b DIR
180    --build-dir=DIR:     Create build here (default: %(WORKDIR)r)
181    --third-party=DIR:   Store third-party sources here (default: %(DEPSRC)r)
182    --sdk-path=DIR:      Location of the SDK (default: %(SDKPATH)r)
183    --src-dir=DIR:       Location of the Python sources (default: %(SRCDIR)r)
184    --dep-target=10.n    OS X deployment target (default: %(DEPTARGET)r)
185    --universal-archs=x  universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r)
186""")% globals()
187
188# Dict of object file names with shared library names to check after building.
189# This is to ensure that we ended up dynamically linking with the shared
190# library paths and versions we expected.  For example:
191#   EXPECTED_SHARED_LIBS['_tkinter.so'] = [
192#                       '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl',
193#                       '/Library/Frameworks/Tk.framework/Versions/8.5/Tk']
194EXPECTED_SHARED_LIBS = {}
195
196# List of names of third party software built with this installer.
197# The names will be inserted into the rtf version of the License.
198THIRD_PARTY_LIBS = []
199
200# Instructions for building libraries that are necessary for building a
201# batteries included python.
202#   [The recipes are defined here for convenience but instantiated later after
203#    command line options have been processed.]
204def library_recipes():
205    result = []
206
207    LT_10_5 = bool(getDeptargetTuple() < (10, 5))
208
209    if not (10, 5) < getDeptargetTuple() < (10, 10):
210        # The OpenSSL libs shipped with OS X 10.5 and earlier are
211        # hopelessly out-of-date and do not include Apple's tie-in to
212        # the root certificates in the user and system keychains via TEA
213        # that was introduced in OS X 10.6.  Note that this applies to
214        # programs built and linked with a 10.5 SDK even when run on
215        # newer versions of OS X.
216        #
217        # Dealing with CAs is messy.  For now, just supply a
218        # local libssl and libcrypto for the older installer variants
219        # (e.g. the python.org 10.5+ 32-bit-only installer) that use the
220        # same default ssl certfile location as the system libs do:
221        #   /System/Library/OpenSSL/cert.pem
222        # Then at least TLS connections can be negotiated with sites that
223        # use sha-256 certs like python.org, assuming the proper CA certs
224        # have been supplied.  The default CA cert management issues for
225        # 10.5 and earlier builds are the same as before, other than it is
226        # now more obvious with cert checking enabled by default in the
227        # standard library.
228        #
229        # For builds with 10.6 through 10.9 SDKs,
230        # continue to use the deprecated but
231        # less out-of-date Apple 0.9.8 libs for now.  While they are less
232        # secure than using an up-to-date 1.0.1 version, doing so
233        # avoids the big problems of forcing users to have to manage
234        # default CAs themselves, thanks to the Apple libs using private TEA
235        # APIs for cert validation from keychains if validation using the
236        # standard OpenSSL locations (/System/Library/OpenSSL, normally empty)
237        # fails.
238        #
239        # Since Apple removed the header files for the deprecated system
240        # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not
241        # have much choice but to build our own copy here, too.
242
243        result.extend([
244          dict(
245              name="OpenSSL 1.0.2j",
246              url="https://www.openssl.org/source/openssl-1.0.2j.tar.gz",
247              checksum='96322138f0b69e61b7212bc53d5e912b',
248              patches=[
249                  "openssl_sdk_makedepend.patch",
250                   ],
251              buildrecipe=build_universal_openssl,
252              configure=None,
253              install=None,
254          ),
255        ])
256
257#   Disable for now
258    if False:   # if getDeptargetTuple() > (10, 5):
259        result.extend([
260          dict(
261              name="Tcl 8.5.15",
262              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tcl8.5.15-src.tar.gz",
263              checksum='f3df162f92c69b254079c4d0af7a690f',
264              buildDir="unix",
265              configure_pre=[
266                    '--enable-shared',
267                    '--enable-threads',
268                    '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
269              ],
270              useLDFlags=False,
271              install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
272                  "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
273                  "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())),
274                  },
275              ),
276          dict(
277              name="Tk 8.5.15",
278              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tk8.5.15-src.tar.gz",
279              checksum='55b8e33f903210a4e1c8bce0f820657f',
280              patches=[
281                  "issue19373_tk_8_5_15_source.patch",
282                   ],
283              buildDir="unix",
284              configure_pre=[
285                    '--enable-aqua',
286                    '--enable-shared',
287                    '--enable-threads',
288                    '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
289              ],
290              useLDFlags=False,
291              install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
292                  "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
293                  "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())),
294                  "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.5'%(getVersion())),
295                  },
296                ),
297        ])
298
299    if PYTHON_3:
300        result.extend([
301          dict(
302              name="XZ 5.0.5",
303              url="http://tukaani.org/xz/xz-5.0.5.tar.gz",
304              checksum='19d924e066b6fff0bc9d1981b4e53196',
305              configure_pre=[
306                    '--disable-dependency-tracking',
307              ]
308              ),
309        ])
310
311    result.extend([
312          dict(
313              name="NCurses 5.9",
314              url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz",
315              checksum='8cb9c412e5f2d96bc6f459aa8c6282a1',
316              configure_pre=[
317                  "--enable-widec",
318                  "--without-cxx",
319                  "--without-cxx-binding",
320                  "--without-ada",
321                  "--without-curses-h",
322                  "--enable-shared",
323                  "--with-shared",
324                  "--without-debug",
325                  "--without-normal",
326                  "--without-tests",
327                  "--without-manpages",
328                  "--datadir=/usr/share",
329                  "--sysconfdir=/etc",
330                  "--sharedstatedir=/usr/com",
331                  "--with-terminfo-dirs=/usr/share/terminfo",
332                  "--with-default-terminfo-dir=/usr/share/terminfo",
333                  "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),),
334              ],
335              patchscripts=[
336                  ("ftp://invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2",
337                   "f54bf02a349f96a7c4f0d00922f3a0d4"),
338                   ],
339              useLDFlags=False,
340              install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%(
341                  shellQuote(os.path.join(WORKDIR, 'libraries')),
342                  shellQuote(os.path.join(WORKDIR, 'libraries')),
343                  getVersion(),
344                  ),
345          ),
346          dict(
347              name="SQLite 3.8.3.1",
348              url="http://www.sqlite.org/2014/sqlite-autoconf-3080301.tar.gz",
349              checksum='509ff98d8dc9729b618b7e96612079c6',
350              extra_cflags=('-Os '
351                            '-DSQLITE_ENABLE_FTS4 '
352                            '-DSQLITE_ENABLE_FTS3_PARENTHESIS '
353                            '-DSQLITE_ENABLE_RTREE '
354                            '-DSQLITE_TCL=0 '
355                 '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]),
356              configure_pre=[
357                  '--enable-threadsafe',
358                  '--enable-shared=no',
359                  '--enable-static=yes',
360                  '--disable-readline',
361                  '--disable-dependency-tracking',
362              ]
363          ),
364        ])
365
366    if getDeptargetTuple() < (10, 5):
367        result.extend([
368          dict(
369              name="Bzip2 1.0.6",
370              url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz",
371              checksum='00b516f4704d4a7cb50a1d97e6e8e15b',
372              configure=None,
373              install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%(
374                  CC, CXX,
375                  shellQuote(os.path.join(WORKDIR, 'libraries')),
376                  ' -arch '.join(ARCHLIST),
377                  SDKPATH,
378              ),
379          ),
380          dict(
381              name="ZLib 1.2.3",
382              url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz",
383              checksum='debc62758716a169df9f62e6ab2bc634',
384              configure=None,
385              install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%(
386                  CC, CXX,
387                  shellQuote(os.path.join(WORKDIR, 'libraries')),
388                  ' -arch '.join(ARCHLIST),
389                  SDKPATH,
390              ),
391          ),
392          dict(
393              # Note that GNU readline is GPL'd software
394              name="GNU Readline 6.1.2",
395              url="http://ftp.gnu.org/pub/gnu/readline/readline-6.1.tar.gz" ,
396              checksum='fc2f7e714fe792db1ce6ddc4c9fb4ef3',
397              patchlevel='0',
398              patches=[
399                  # The readline maintainers don't do actual micro releases, but
400                  # just ship a set of patches.
401                  ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-001',
402                   'c642f2e84d820884b0bf9fd176bc6c3f'),
403                  ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-002',
404                   '1a76781a1ea734e831588285db7ec9b1'),
405              ]
406          ),
407        ])
408
409    if not PYTHON_3:
410        result.extend([
411          dict(
412              name="Sleepycat DB 4.7.25",
413              url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz",
414              checksum='ec2b87e833779681a0c3a814aa71359e',
415              buildDir="build_unix",
416              configure="../dist/configure",
417              configure_pre=[
418                  '--includedir=/usr/local/include/db4',
419              ]
420          ),
421        ])
422
423    return result
424
425
426# Instructions for building packages inside the .mpkg.
427def pkg_recipes():
428    unselected_for_python3 = ('selected', 'unselected')[PYTHON_3]
429    result = [
430        dict(
431            name="PythonFramework",
432            long_name="Python Framework",
433            source="/Library/Frameworks/Python.framework",
434            readme="""\
435                This package installs Python.framework, that is the python
436                interpreter and the standard library. This also includes Python
437                wrappers for lots of Mac OS X API's.
438            """,
439            postflight="scripts/postflight.framework",
440            selected='selected',
441        ),
442        dict(
443            name="PythonApplications",
444            long_name="GUI Applications",
445            source="/Applications/Python %(VER)s",
446            readme="""\
447                This package installs IDLE (an interactive Python IDE),
448                Python Launcher and Build Applet (create application bundles
449                from python scripts).
450
451                It also installs a number of examples and demos.
452                """,
453            required=False,
454            selected='selected',
455        ),
456        dict(
457            name="PythonUnixTools",
458            long_name="UNIX command-line tools",
459            source="/usr/local/bin",
460            readme="""\
461                This package installs the unix tools in /usr/local/bin for
462                compatibility with older releases of Python. This package
463                is not necessary to use Python.
464                """,
465            required=False,
466            selected='selected',
467        ),
468        dict(
469            name="PythonDocumentation",
470            long_name="Python Documentation",
471            topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation",
472            source="/pydocs",
473            readme="""\
474                This package installs the python documentation at a location
475                that is useable for pydoc and IDLE.
476                """,
477            postflight="scripts/postflight.documentation",
478            required=False,
479            selected='selected',
480        ),
481        dict(
482            name="PythonProfileChanges",
483            long_name="Shell profile updater",
484            readme="""\
485                This packages updates your shell profile to make sure that
486                the Python tools are found by your shell in preference of
487                the system provided Python tools.
488
489                If you don't install this package you'll have to add
490                "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin"
491                to your PATH by hand.
492                """,
493            postflight="scripts/postflight.patch-profile",
494            topdir="/Library/Frameworks/Python.framework",
495            source="/empty-dir",
496            required=False,
497            selected='selected',
498        ),
499        dict(
500            name="PythonInstallPip",
501            long_name="Install or upgrade pip",
502            readme="""\
503                This package installs (or upgrades from an earlier version)
504                pip, a tool for installing and managing Python packages.
505                """,
506            postflight="scripts/postflight.ensurepip",
507            topdir="/Library/Frameworks/Python.framework",
508            source="/empty-dir",
509            required=False,
510            selected='selected',
511        ),
512    ]
513
514    if getDeptargetTuple() < (10, 4) and not PYTHON_3:
515        result.append(
516            dict(
517                name="PythonSystemFixes",
518                long_name="Fix system Python",
519                readme="""\
520                    This package updates the system python installation on
521                    Mac OS X 10.3 to ensure that you can build new python extensions
522                    using that copy of python after installing this version.
523                    """,
524                postflight="../Tools/fixapplepython23.py",
525                topdir="/Library/Frameworks/Python.framework",
526                source="/empty-dir",
527                required=False,
528                selected=unselected_for_python3,
529            )
530        )
531
532    return result
533
534def fatal(msg):
535    """
536    A fatal error, bail out.
537    """
538    sys.stderr.write('FATAL: ')
539    sys.stderr.write(msg)
540    sys.stderr.write('\n')
541    sys.exit(1)
542
543def fileContents(fn):
544    """
545    Return the contents of the named file
546    """
547    return open(fn, 'r').read()
548
549def runCommand(commandline):
550    """
551    Run a command and raise RuntimeError if it fails. Output is suppressed
552    unless the command fails.
553    """
554    fd = os.popen(commandline, 'r')
555    data = fd.read()
556    xit = fd.close()
557    if xit is not None:
558        sys.stdout.write(data)
559        raise RuntimeError("command failed: %s"%(commandline,))
560
561    if VERBOSE:
562        sys.stdout.write(data); sys.stdout.flush()
563
564def captureCommand(commandline):
565    fd = os.popen(commandline, 'r')
566    data = fd.read()
567    xit = fd.close()
568    if xit is not None:
569        sys.stdout.write(data)
570        raise RuntimeError("command failed: %s"%(commandline,))
571
572    return data
573
574def getTclTkVersion(configfile, versionline):
575    """
576    search Tcl or Tk configuration file for version line
577    """
578    try:
579        f = open(configfile, "r")
580    except:
581        fatal("Framework configuration file not found: %s" % configfile)
582
583    for l in f:
584        if l.startswith(versionline):
585            f.close()
586            return l
587
588    fatal("Version variable %s not found in framework configuration file: %s"
589            % (versionline, configfile))
590
591def checkEnvironment():
592    """
593    Check that we're running on a supported system.
594    """
595
596    if sys.version_info[0:2] < (2, 4):
597        fatal("This script must be run with Python 2.4 or later")
598
599    if platform.system() != 'Darwin':
600        fatal("This script should be run on a Mac OS X 10.4 (or later) system")
601
602    if int(platform.release().split('.')[0]) < 8:
603        fatal("This script should be run on a Mac OS X 10.4 (or later) system")
604
605    if not os.path.exists(SDKPATH):
606        fatal("Please install the latest version of Xcode and the %s SDK"%(
607            os.path.basename(SDKPATH[:-4])))
608
609    # Because we only support dynamic load of only one major/minor version of
610    # Tcl/Tk, ensure:
611    # 1. there are no user-installed frameworks of Tcl/Tk with version
612    #       higher than the Apple-supplied system version in
613    #       SDKROOT/System/Library/Frameworks
614    # 2. there is a user-installed framework (usually ActiveTcl) in (or linked
615    #       in) SDKROOT/Library/Frameworks with the same version as the system
616    #       version. This allows users to choose to install a newer patch level.
617
618    frameworks = {}
619    for framework in ['Tcl', 'Tk']:
620        fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework
621        sysfw = os.path.join(SDKPATH, 'System', fwpth)
622        libfw = os.path.join(SDKPATH, fwpth)
623        usrfw = os.path.join(os.getenv('HOME'), fwpth)
624        frameworks[framework] = os.readlink(sysfw)
625        if not os.path.exists(libfw):
626            fatal("Please install a link to a current %s %s as %s so "
627                    "the user can override the system framework."
628                    % (framework, frameworks[framework], libfw))
629        if os.readlink(libfw) != os.readlink(sysfw):
630            fatal("Version of %s must match %s" % (libfw, sysfw) )
631        if os.path.exists(usrfw):
632            fatal("Please rename %s to avoid possible dynamic load issues."
633                    % usrfw)
634
635    if frameworks['Tcl'] != frameworks['Tk']:
636        fatal("The Tcl and Tk frameworks are not the same version.")
637
638    # add files to check after build
639    EXPECTED_SHARED_LIBS['_tkinter.so'] = [
640            "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl"
641                % frameworks['Tcl'],
642            "/Library/Frameworks/Tk.framework/Versions/%s/Tk"
643                % frameworks['Tk'],
644            ]
645
646    # Remove inherited environment variables which might influence build
647    environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_',
648                            'LD_', 'LIBRARY_', 'PATH', 'PYTHON']
649    for ev in list(os.environ):
650        for prefix in environ_var_prefixes:
651            if ev.startswith(prefix) :
652                print("INFO: deleting environment variable %s=%s" % (
653                                                    ev, os.environ[ev]))
654                del os.environ[ev]
655
656    base_path = '/bin:/sbin:/usr/bin:/usr/sbin'
657    if 'SDK_TOOLS_BIN' in os.environ:
658        base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path
659    # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin;
660    # add its fixed location here if it exists
661    OLD_DEVELOPER_TOOLS = '/Developer/Tools'
662    if os.path.isdir(OLD_DEVELOPER_TOOLS):
663        base_path = base_path + ':' + OLD_DEVELOPER_TOOLS
664    os.environ['PATH'] = base_path
665    print("Setting default PATH: %s"%(os.environ['PATH']))
666    # Ensure ws have access to hg and to sphinx-build.
667    # You may have to create links in /usr/bin for them.
668    runCommand('hg --version')
669    runCommand('sphinx-build --version')
670
671def parseOptions(args=None):
672    """
673    Parse arguments and update global settings.
674    """
675    global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET
676    global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX
677    global FW_VERSION_PREFIX
678
679    if args is None:
680        args = sys.argv[1:]
681
682    try:
683        options, args = getopt.getopt(args, '?hb',
684                [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=',
685                  'dep-target=', 'universal-archs=', 'help' ])
686    except getopt.GetoptError:
687        print(sys.exc_info()[1])
688        sys.exit(1)
689
690    if args:
691        print("Additional arguments")
692        sys.exit(1)
693
694    deptarget = None
695    for k, v in options:
696        if k in ('-h', '-?', '--help'):
697            print(USAGE)
698            sys.exit(0)
699
700        elif k in ('-d', '--build-dir'):
701            WORKDIR=v
702
703        elif k in ('--third-party',):
704            DEPSRC=v
705
706        elif k in ('--sdk-path',):
707            SDKPATH=v
708
709        elif k in ('--src-dir',):
710            SRCDIR=v
711
712        elif k in ('--dep-target', ):
713            DEPTARGET=v
714            deptarget=v
715
716        elif k in ('--universal-archs', ):
717            if v in UNIVERSALOPTS:
718                UNIVERSALARCHS = v
719                ARCHLIST = universal_opts_map[UNIVERSALARCHS]
720                if deptarget is None:
721                    # Select alternate default deployment
722                    # target
723                    DEPTARGET = default_target_map.get(v, '10.3')
724            else:
725                raise NotImplementedError(v)
726
727        else:
728            raise NotImplementedError(k)
729
730    SRCDIR=os.path.abspath(SRCDIR)
731    WORKDIR=os.path.abspath(WORKDIR)
732    SDKPATH=os.path.abspath(SDKPATH)
733    DEPSRC=os.path.abspath(DEPSRC)
734
735    CC, CXX = getTargetCompilers()
736
737    FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()]
738
739    print("-- Settings:")
740    print("   * Source directory:    %s" % SRCDIR)
741    print("   * Build directory:     %s" % WORKDIR)
742    print("   * SDK location:        %s" % SDKPATH)
743    print("   * Third-party source:  %s" % DEPSRC)
744    print("   * Deployment target:   %s" % DEPTARGET)
745    print("   * Universal archs:     %s" % str(ARCHLIST))
746    print("   * C compiler:          %s" % CC)
747    print("   * C++ compiler:        %s" % CXX)
748    print("")
749    print(" -- Building a Python %s framework at patch level %s"
750                % (getVersion(), getFullVersion()))
751    print("")
752
753def extractArchive(builddir, archiveName):
754    """
755    Extract a source archive into 'builddir'. Returns the path of the
756    extracted archive.
757
758    XXX: This function assumes that archives contain a toplevel directory
759    that is has the same name as the basename of the archive. This is
760    safe enough for almost anything we use.  Unfortunately, it does not
761    work for current Tcl and Tk source releases where the basename of
762    the archive ends with "-src" but the uncompressed directory does not.
763    For now, just special case Tcl and Tk tar.gz downloads.
764    """
765    curdir = os.getcwd()
766    try:
767        os.chdir(builddir)
768        if archiveName.endswith('.tar.gz'):
769            retval = os.path.basename(archiveName[:-7])
770            if ((retval.startswith('tcl') or retval.startswith('tk'))
771                    and retval.endswith('-src')):
772                retval = retval[:-4]
773            if os.path.exists(retval):
774                shutil.rmtree(retval)
775            fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r')
776
777        elif archiveName.endswith('.tar.bz2'):
778            retval = os.path.basename(archiveName[:-8])
779            if os.path.exists(retval):
780                shutil.rmtree(retval)
781            fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r')
782
783        elif archiveName.endswith('.tar'):
784            retval = os.path.basename(archiveName[:-4])
785            if os.path.exists(retval):
786                shutil.rmtree(retval)
787            fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r')
788
789        elif archiveName.endswith('.zip'):
790            retval = os.path.basename(archiveName[:-4])
791            if os.path.exists(retval):
792                shutil.rmtree(retval)
793            fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r')
794
795        data = fp.read()
796        xit = fp.close()
797        if xit is not None:
798            sys.stdout.write(data)
799            raise RuntimeError("Cannot extract %s"%(archiveName,))
800
801        return os.path.join(builddir, retval)
802
803    finally:
804        os.chdir(curdir)
805
806def downloadURL(url, fname):
807    """
808    Download the contents of the url into the file.
809    """
810    fpIn = urllib_request.urlopen(url)
811    fpOut = open(fname, 'wb')
812    block = fpIn.read(10240)
813    try:
814        while block:
815            fpOut.write(block)
816            block = fpIn.read(10240)
817        fpIn.close()
818        fpOut.close()
819    except:
820        try:
821            os.unlink(fname)
822        except:
823            pass
824
825def verifyThirdPartyFile(url, checksum, fname):
826    """
827    Download file from url to filename fname if it does not already exist.
828    Abort if file contents does not match supplied md5 checksum.
829    """
830    name = os.path.basename(fname)
831    if os.path.exists(fname):
832        print("Using local copy of %s"%(name,))
833    else:
834        print("Did not find local copy of %s"%(name,))
835        print("Downloading %s"%(name,))
836        downloadURL(url, fname)
837        print("Archive for %s stored as %s"%(name, fname))
838    if os.system(
839            'MD5=$(openssl md5 %s) ; test "${MD5##*= }" = "%s"'
840                % (shellQuote(fname), checksum) ):
841        fatal('MD5 checksum mismatch for file %s' % fname)
842
843def build_universal_openssl(basedir, archList):
844    """
845    Special case build recipe for universal build of openssl.
846
847    The upstream OpenSSL build system does not directly support
848    OS X universal builds.  We need to build each architecture
849    separately then lipo them together into fat libraries.
850    """
851
852    # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4).
853    # If we are building on a 10.4.x or earlier system,
854    # unilaterally disable assembly code building to avoid the problem.
855    no_asm = int(platform.release().split(".")[0]) < 9
856
857    def build_openssl_arch(archbase, arch):
858        "Build one architecture of openssl"
859        arch_opts = {
860            "i386": ["darwin-i386-cc"],
861            "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"],
862            "ppc": ["darwin-ppc-cc"],
863            "ppc64": ["darwin64-ppc-cc"],
864        }
865        configure_opts = [
866            "no-krb5",
867            "no-idea",
868            "no-mdc2",
869            "no-rc5",
870            "no-zlib",
871            "enable-tlsext",
872            "no-ssl2",
873            "no-ssl3",
874            "no-ssl3-method",
875            # "enable-unit-test",
876            "shared",
877            "--install_prefix=%s"%shellQuote(archbase),
878            "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX),
879            "--openssldir=/System/Library/OpenSSL",
880        ]
881        if no_asm:
882            configure_opts.append("no-asm")
883        runCommand(" ".join(["perl", "Configure"]
884                        + arch_opts[arch] + configure_opts))
885        runCommand("make depend OSX_SDK=%s" % SDKPATH)
886        runCommand("make all OSX_SDK=%s" % SDKPATH)
887        runCommand("make install_sw OSX_SDK=%s" % SDKPATH)
888        # runCommand("make test")
889        return
890
891    srcdir = os.getcwd()
892    universalbase = os.path.join(srcdir, "..",
893                        os.path.basename(srcdir) + "-universal")
894    os.mkdir(universalbase)
895    archbasefws = []
896    for arch in archList:
897        # fresh copy of the source tree
898        archsrc = os.path.join(universalbase, arch, "src")
899        shutil.copytree(srcdir, archsrc, symlinks=True)
900        # install base for this arch
901        archbase = os.path.join(universalbase, arch, "root")
902        os.mkdir(archbase)
903        # Python framework base within install_prefix:
904        # the build will install into this framework..
905        # This is to ensure that the resulting shared libs have
906        # the desired real install paths built into them.
907        archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX)
908
909        # build one architecture
910        os.chdir(archsrc)
911        build_openssl_arch(archbase, arch)
912        os.chdir(srcdir)
913        archbasefws.append(archbasefw)
914
915    # copy arch-independent files from last build into the basedir framework
916    basefw = os.path.join(basedir, *FW_VERSION_PREFIX)
917    shutil.copytree(
918            os.path.join(archbasefw, "include", "openssl"),
919            os.path.join(basefw, "include", "openssl")
920            )
921
922    shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"),
923            "SHLIB_VERSION_NUMBER")
924    #   e.g. -> "1.0.0"
925    libcrypto = "libcrypto.dylib"
926    libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".")
927    #   e.g. -> "libcrypto.1.0.0.dylib"
928    libssl = "libssl.dylib"
929    libssl_versioned = libssl.replace(".", "."+shlib_version_number+".")
930    #   e.g. -> "libssl.1.0.0.dylib"
931
932    try:
933        os.mkdir(os.path.join(basefw, "lib"))
934    except OSError:
935        pass
936
937    # merge the individual arch-dependent shared libs into a fat shared lib
938    archbasefws.insert(0, basefw)
939    for (lib_unversioned, lib_versioned) in [
940                (libcrypto, libcrypto_versioned),
941                (libssl, libssl_versioned)
942            ]:
943        runCommand("lipo -create -output " +
944                    " ".join(shellQuote(
945                            os.path.join(fw, "lib", lib_versioned))
946                                    for fw in archbasefws))
947        # and create an unversioned symlink of it
948        os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned))
949
950    # Create links in the temp include and lib dirs that will be injected
951    # into the Python build so that setup.py can find them while building
952    # and the versioned links so that the setup.py post-build import test
953    # does not fail.
954    relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX)
955    for fn in [
956            ["include", "openssl"],
957            ["lib", libcrypto],
958            ["lib", libssl],
959            ["lib", libcrypto_versioned],
960            ["lib", libssl_versioned],
961        ]:
962        os.symlink(
963            os.path.join(relative_path, *fn),
964            os.path.join(basedir, "usr", "local", *fn)
965        )
966
967    return
968
969def buildRecipe(recipe, basedir, archList):
970    """
971    Build software using a recipe. This function does the
972    'configure;make;make install' dance for C software, with a possibility
973    to customize this process, basically a poor-mans DarwinPorts.
974    """
975    curdir = os.getcwd()
976
977    name = recipe['name']
978    THIRD_PARTY_LIBS.append(name)
979    url = recipe['url']
980    configure = recipe.get('configure', './configure')
981    buildrecipe = recipe.get('buildrecipe', None)
982    install = recipe.get('install', 'make && make install DESTDIR=%s'%(
983        shellQuote(basedir)))
984
985    archiveName = os.path.split(url)[-1]
986    sourceArchive = os.path.join(DEPSRC, archiveName)
987
988    if not os.path.exists(DEPSRC):
989        os.mkdir(DEPSRC)
990
991    verifyThirdPartyFile(url, recipe['checksum'], sourceArchive)
992    print("Extracting archive for %s"%(name,))
993    buildDir=os.path.join(WORKDIR, '_bld')
994    if not os.path.exists(buildDir):
995        os.mkdir(buildDir)
996
997    workDir = extractArchive(buildDir, sourceArchive)
998    os.chdir(workDir)
999
1000    for patch in recipe.get('patches', ()):
1001        if isinstance(patch, tuple):
1002            url, checksum = patch
1003            fn = os.path.join(DEPSRC, os.path.basename(url))
1004            verifyThirdPartyFile(url, checksum, fn)
1005        else:
1006            # patch is a file in the source directory
1007            fn = os.path.join(curdir, patch)
1008        runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1),
1009            shellQuote(fn),))
1010
1011    for patchscript in recipe.get('patchscripts', ()):
1012        if isinstance(patchscript, tuple):
1013            url, checksum = patchscript
1014            fn = os.path.join(DEPSRC, os.path.basename(url))
1015            verifyThirdPartyFile(url, checksum, fn)
1016        else:
1017            # patch is a file in the source directory
1018            fn = os.path.join(curdir, patchscript)
1019        if fn.endswith('.bz2'):
1020            runCommand('bunzip2 -fk %s' % shellQuote(fn))
1021            fn = fn[:-4]
1022        runCommand('sh %s' % shellQuote(fn))
1023        os.unlink(fn)
1024
1025    if 'buildDir' in recipe:
1026        os.chdir(recipe['buildDir'])
1027
1028    if configure is not None:
1029        configure_args = [
1030            "--prefix=/usr/local",
1031            "--enable-static",
1032            "--disable-shared",
1033            #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),),
1034        ]
1035
1036        if 'configure_pre' in recipe:
1037            args = list(recipe['configure_pre'])
1038            if '--disable-static' in args:
1039                configure_args.remove('--enable-static')
1040            if '--enable-shared' in args:
1041                configure_args.remove('--disable-shared')
1042            configure_args.extend(args)
1043
1044        if recipe.get('useLDFlags', 1):
1045            configure_args.extend([
1046                "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s "
1047                            "-I%s/usr/local/include"%(
1048                        recipe.get('extra_cflags', ''),
1049                        DEPTARGET,
1050                        ' -arch '.join(archList),
1051                        shellQuote(SDKPATH)[1:-1],
1052                        shellQuote(basedir)[1:-1],),
1053                "LDFLAGS=-mmacosx-version-min=%s -isysroot %s -L%s/usr/local/lib -arch %s"%(
1054                    DEPTARGET,
1055                    shellQuote(SDKPATH)[1:-1],
1056                    shellQuote(basedir)[1:-1],
1057                    ' -arch '.join(archList)),
1058            ])
1059        else:
1060            configure_args.extend([
1061                "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s "
1062                            "-I%s/usr/local/include"%(
1063                        recipe.get('extra_cflags', ''),
1064                        DEPTARGET,
1065                        ' -arch '.join(archList),
1066                        shellQuote(SDKPATH)[1:-1],
1067                        shellQuote(basedir)[1:-1],),
1068            ])
1069
1070        if 'configure_post' in recipe:
1071            configure_args = configure_args + list(recipe['configure_post'])
1072
1073        configure_args.insert(0, configure)
1074        configure_args = [ shellQuote(a) for a in configure_args ]
1075
1076        print("Running configure for %s"%(name,))
1077        runCommand(' '.join(configure_args) + ' 2>&1')
1078
1079    if buildrecipe is not None:
1080        # call special-case build recipe, e.g. for openssl
1081        buildrecipe(basedir, archList)
1082
1083    if install is not None:
1084        print("Running install for %s"%(name,))
1085        runCommand('{ ' + install + ' ;} 2>&1')
1086
1087    print("Done %s"%(name,))
1088    print("")
1089
1090    os.chdir(curdir)
1091
1092def buildLibraries():
1093    """
1094    Build our dependencies into $WORKDIR/libraries/usr/local
1095    """
1096    print("")
1097    print("Building required libraries")
1098    print("")
1099    universal = os.path.join(WORKDIR, 'libraries')
1100    os.mkdir(universal)
1101    os.makedirs(os.path.join(universal, 'usr', 'local', 'lib'))
1102    os.makedirs(os.path.join(universal, 'usr', 'local', 'include'))
1103
1104    for recipe in library_recipes():
1105        buildRecipe(recipe, universal, ARCHLIST)
1106
1107
1108
1109def buildPythonDocs():
1110    # This stores the documentation as Resources/English.lproj/Documentation
1111    # inside the framwork. pydoc and IDLE will pick it up there.
1112    print("Install python documentation")
1113    rootDir = os.path.join(WORKDIR, '_root')
1114    buildDir = os.path.join('../../Doc')
1115    docdir = os.path.join(rootDir, 'pydocs')
1116    curDir = os.getcwd()
1117    os.chdir(buildDir)
1118    # The Doc build changed for 3.4 (technically, for 3.4.1) and for 2.7.9
1119    runCommand('make clean')
1120    # Assume sphinx-build is on our PATH, checked in checkEnvironment
1121    runCommand('make html')
1122    os.chdir(curDir)
1123    if not os.path.exists(docdir):
1124        os.mkdir(docdir)
1125    os.rename(os.path.join(buildDir, 'build', 'html'), docdir)
1126
1127
1128def buildPython():
1129    print("Building a universal python for %s architectures" % UNIVERSALARCHS)
1130
1131    buildDir = os.path.join(WORKDIR, '_bld', 'python')
1132    rootDir = os.path.join(WORKDIR, '_root')
1133
1134    if os.path.exists(buildDir):
1135        shutil.rmtree(buildDir)
1136    if os.path.exists(rootDir):
1137        shutil.rmtree(rootDir)
1138    os.makedirs(buildDir)
1139    os.makedirs(rootDir)
1140    os.makedirs(os.path.join(rootDir, 'empty-dir'))
1141    curdir = os.getcwd()
1142    os.chdir(buildDir)
1143
1144    # Not sure if this is still needed, the original build script
1145    # claims that parts of the install assume python.exe exists.
1146    os.symlink('python', os.path.join(buildDir, 'python.exe'))
1147
1148    # Extract the version from the configure file, needed to calculate
1149    # several paths.
1150    version = getVersion()
1151
1152    # Since the extra libs are not in their installed framework location
1153    # during the build, augment the library path so that the interpreter
1154    # will find them during its extension import sanity checks.
1155    os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR,
1156                                        'libraries', 'usr', 'local', 'lib')
1157    print("Running configure...")
1158    runCommand("%s -C --enable-framework --enable-universalsdk=%s "
1159               "--with-universal-archs=%s "
1160               "%s "
1161               "%s "
1162               "LDFLAGS='-g -L%s/libraries/usr/local/lib' "
1163               "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%(
1164        shellQuote(os.path.join(SRCDIR, 'configure')), shellQuote(SDKPATH),
1165        UNIVERSALARCHS,
1166        (' ', '--with-computed-gotos ')[PYTHON_3],
1167        (' ', '--without-ensurepip ')[PYTHON_3],
1168        shellQuote(WORKDIR)[1:-1],
1169        shellQuote(WORKDIR)[1:-1]))
1170
1171    print("Running make touch")
1172    runCommand("make touch")
1173
1174    print("Running make")
1175    runCommand("make")
1176
1177    print("Running make install")
1178    runCommand("make install DESTDIR=%s"%(
1179        shellQuote(rootDir)))
1180
1181    print("Running make frameworkinstallextras")
1182    runCommand("make frameworkinstallextras DESTDIR=%s"%(
1183        shellQuote(rootDir)))
1184
1185    del os.environ['DYLD_LIBRARY_PATH']
1186    print("Copying required shared libraries")
1187    if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')):
1188        runCommand("mv %s/* %s"%(
1189            shellQuote(os.path.join(
1190                WORKDIR, 'libraries', 'Library', 'Frameworks',
1191                'Python.framework', 'Versions', getVersion(),
1192                'lib')),
1193            shellQuote(os.path.join(WORKDIR, '_root', 'Library', 'Frameworks',
1194                'Python.framework', 'Versions', getVersion(),
1195                'lib'))))
1196
1197    path_to_lib = os.path.join(rootDir, 'Library', 'Frameworks',
1198                                'Python.framework', 'Versions',
1199                                version, 'lib', 'python%s'%(version,))
1200
1201    print("Fix file modes")
1202    frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework')
1203    gid = grp.getgrnam('admin').gr_gid
1204
1205    shared_lib_error = False
1206    for dirpath, dirnames, filenames in os.walk(frmDir):
1207        for dn in dirnames:
1208            os.chmod(os.path.join(dirpath, dn), STAT_0o775)
1209            os.chown(os.path.join(dirpath, dn), -1, gid)
1210
1211        for fn in filenames:
1212            if os.path.islink(fn):
1213                continue
1214
1215            # "chmod g+w $fn"
1216            p = os.path.join(dirpath, fn)
1217            st = os.stat(p)
1218            os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP)
1219            os.chown(p, -1, gid)
1220
1221            if fn in EXPECTED_SHARED_LIBS:
1222                # check to see that this file was linked with the
1223                # expected library path and version
1224                data = captureCommand("otool -L %s" % shellQuote(p))
1225                for sl in EXPECTED_SHARED_LIBS[fn]:
1226                    if ("\t%s " % sl) not in data:
1227                        print("Expected shared lib %s was not linked with %s"
1228                                % (sl, p))
1229                        shared_lib_error = True
1230
1231    if shared_lib_error:
1232        fatal("Unexpected shared library errors.")
1233
1234    if PYTHON_3:
1235        LDVERSION=None
1236        VERSION=None
1237        ABIFLAGS=None
1238
1239        fp = open(os.path.join(buildDir, 'Makefile'), 'r')
1240        for ln in fp:
1241            if ln.startswith('VERSION='):
1242                VERSION=ln.split()[1]
1243            if ln.startswith('ABIFLAGS='):
1244                ABIFLAGS=ln.split()[1]
1245            if ln.startswith('LDVERSION='):
1246                LDVERSION=ln.split()[1]
1247        fp.close()
1248
1249        LDVERSION = LDVERSION.replace('$(VERSION)', VERSION)
1250        LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS)
1251        config_suffix = '-' + LDVERSION
1252    else:
1253        config_suffix = ''      # Python 2.x
1254
1255    # We added some directories to the search path during the configure
1256    # phase. Remove those because those directories won't be there on
1257    # the end-users system. Also remove the directories from _sysconfigdata.py
1258    # (added in 3.3) if it exists.
1259
1260    include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,)
1261    lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,)
1262
1263    # fix Makefile
1264    path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile')
1265    fp = open(path, 'r')
1266    data = fp.read()
1267    fp.close()
1268
1269    for p in (include_path, lib_path):
1270        data = data.replace(" " + p, '')
1271        data = data.replace(p + " ", '')
1272
1273    fp = open(path, 'w')
1274    fp.write(data)
1275    fp.close()
1276
1277    # fix _sysconfigdata if it exists
1278    #
1279    # TODO: make this more robust!  test_sysconfig_module of
1280    # distutils.tests.test_sysconfig.SysconfigTestCase tests that
1281    # the output from get_config_var in both sysconfig and
1282    # distutils.sysconfig is exactly the same for both CFLAGS and
1283    # LDFLAGS.  The fixing up is now complicated by the pretty
1284    # printing in _sysconfigdata.py.  Also, we are using the
1285    # pprint from the Python running the installer build which
1286    # may not cosmetically format the same as the pprint in the Python
1287    # being built (and which is used to originally generate
1288    # _sysconfigdata.py).
1289
1290    import pprint
1291    path = os.path.join(path_to_lib, '_sysconfigdata.py')
1292    if os.path.exists(path):
1293        fp = open(path, 'r')
1294        data = fp.read()
1295        fp.close()
1296        # create build_time_vars dict
1297        exec(data)
1298        vars = {}
1299        for k, v in build_time_vars.items():
1300            if type(v) == type(''):
1301                for p in (include_path, lib_path):
1302                    v = v.replace(' ' + p, '')
1303                    v = v.replace(p + ' ', '')
1304            vars[k] = v
1305
1306        fp = open(path, 'w')
1307        # duplicated from sysconfig._generate_posix_vars()
1308        fp.write('# system configuration generated and used by'
1309                    ' the sysconfig module\n')
1310        fp.write('build_time_vars = ')
1311        pprint.pprint(vars, stream=fp)
1312        fp.close()
1313
1314    # Add symlinks in /usr/local/bin, using relative links
1315    usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin')
1316    to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks',
1317            'Python.framework', 'Versions', version, 'bin')
1318    if os.path.exists(usr_local_bin):
1319        shutil.rmtree(usr_local_bin)
1320    os.makedirs(usr_local_bin)
1321    for fn in os.listdir(
1322                os.path.join(frmDir, 'Versions', version, 'bin')):
1323        os.symlink(os.path.join(to_framework, fn),
1324                   os.path.join(usr_local_bin, fn))
1325
1326    os.chdir(curdir)
1327
1328    if PYTHON_3:
1329        # Remove the 'Current' link, that way we don't accidentally mess
1330        # with an already installed version of python 2
1331        os.unlink(os.path.join(rootDir, 'Library', 'Frameworks',
1332                            'Python.framework', 'Versions', 'Current'))
1333
1334def patchFile(inPath, outPath):
1335    data = fileContents(inPath)
1336    data = data.replace('$FULL_VERSION', getFullVersion())
1337    data = data.replace('$VERSION', getVersion())
1338    data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later')))
1339    data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS]))
1340    data = data.replace('$INSTALL_SIZE', installSize())
1341    data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS))
1342
1343    # This one is not handy as a template variable
1344    data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework')
1345    fp = open(outPath, 'w')
1346    fp.write(data)
1347    fp.close()
1348
1349def patchScript(inPath, outPath):
1350    major, minor = getVersionMajorMinor()
1351    data = fileContents(inPath)
1352    data = data.replace('@PYMAJOR@', str(major))
1353    data = data.replace('@PYVER@', getVersion())
1354    fp = open(outPath, 'w')
1355    fp.write(data)
1356    fp.close()
1357    os.chmod(outPath, STAT_0o755)
1358
1359
1360
1361def packageFromRecipe(targetDir, recipe):
1362    curdir = os.getcwd()
1363    try:
1364        # The major version (such as 2.5) is included in the package name
1365        # because having two version of python installed at the same time is
1366        # common.
1367        pkgname = '%s-%s'%(recipe['name'], getVersion())
1368        srcdir  = recipe.get('source')
1369        pkgroot = recipe.get('topdir', srcdir)
1370        postflight = recipe.get('postflight')
1371        readme = textwrap.dedent(recipe['readme'])
1372        isRequired = recipe.get('required', True)
1373
1374        print("- building package %s"%(pkgname,))
1375
1376        # Substitute some variables
1377        textvars = dict(
1378            VER=getVersion(),
1379            FULLVER=getFullVersion(),
1380        )
1381        readme = readme % textvars
1382
1383        if pkgroot is not None:
1384            pkgroot = pkgroot % textvars
1385        else:
1386            pkgroot = '/'
1387
1388        if srcdir is not None:
1389            srcdir = os.path.join(WORKDIR, '_root', srcdir[1:])
1390            srcdir = srcdir % textvars
1391
1392        if postflight is not None:
1393            postflight = os.path.abspath(postflight)
1394
1395        packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents')
1396        os.makedirs(packageContents)
1397
1398        if srcdir is not None:
1399            os.chdir(srcdir)
1400            runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),))
1401            runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),))
1402            runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),))
1403
1404        fn = os.path.join(packageContents, 'PkgInfo')
1405        fp = open(fn, 'w')
1406        fp.write('pmkrpkg1')
1407        fp.close()
1408
1409        rsrcDir = os.path.join(packageContents, "Resources")
1410        os.mkdir(rsrcDir)
1411        fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w')
1412        fp.write(readme)
1413        fp.close()
1414
1415        if postflight is not None:
1416            patchScript(postflight, os.path.join(rsrcDir, 'postflight'))
1417
1418        vers = getFullVersion()
1419        major, minor = getVersionMajorMinor()
1420        pl = Plist(
1421                CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,),
1422                CFBundleIdentifier='org.python.Python.%s'%(pkgname,),
1423                CFBundleName='Python.%s'%(pkgname,),
1424                CFBundleShortVersionString=vers,
1425                IFMajorVersion=major,
1426                IFMinorVersion=minor,
1427                IFPkgFormatVersion=0.10000000149011612,
1428                IFPkgFlagAllowBackRev=False,
1429                IFPkgFlagAuthorizationAction="RootAuthorization",
1430                IFPkgFlagDefaultLocation=pkgroot,
1431                IFPkgFlagFollowLinks=True,
1432                IFPkgFlagInstallFat=True,
1433                IFPkgFlagIsRequired=isRequired,
1434                IFPkgFlagOverwritePermissions=False,
1435                IFPkgFlagRelocatable=False,
1436                IFPkgFlagRestartAction="NoRestart",
1437                IFPkgFlagRootVolumeOnly=True,
1438                IFPkgFlagUpdateInstalledLangauges=False,
1439            )
1440        writePlist(pl, os.path.join(packageContents, 'Info.plist'))
1441
1442        pl = Plist(
1443                    IFPkgDescriptionDescription=readme,
1444                    IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)),
1445                    IFPkgDescriptionVersion=vers,
1446                )
1447        writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist'))
1448
1449    finally:
1450        os.chdir(curdir)
1451
1452
1453def makeMpkgPlist(path):
1454
1455    vers = getFullVersion()
1456    major, minor = getVersionMajorMinor()
1457
1458    pl = Plist(
1459            CFBundleGetInfoString="Python %s"%(vers,),
1460            CFBundleIdentifier='org.python.Python',
1461            CFBundleName='Python',
1462            CFBundleShortVersionString=vers,
1463            IFMajorVersion=major,
1464            IFMinorVersion=minor,
1465            IFPkgFlagComponentDirectory="Contents/Packages",
1466            IFPkgFlagPackageList=[
1467                dict(
1468                    IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()),
1469                    IFPkgFlagPackageSelection=item.get('selected', 'selected'),
1470                )
1471                for item in pkg_recipes()
1472            ],
1473            IFPkgFormatVersion=0.10000000149011612,
1474            IFPkgFlagBackgroundScaling="proportional",
1475            IFPkgFlagBackgroundAlignment="left",
1476            IFPkgFlagAuthorizationAction="RootAuthorization",
1477        )
1478
1479    writePlist(pl, path)
1480
1481
1482def buildInstaller():
1483
1484    # Zap all compiled files
1485    for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')):
1486        for fn in filenames:
1487            if fn.endswith('.pyc') or fn.endswith('.pyo'):
1488                os.unlink(os.path.join(dirpath, fn))
1489
1490    outdir = os.path.join(WORKDIR, 'installer')
1491    if os.path.exists(outdir):
1492        shutil.rmtree(outdir)
1493    os.mkdir(outdir)
1494
1495    pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents')
1496    pkgcontents = os.path.join(pkgroot, 'Packages')
1497    os.makedirs(pkgcontents)
1498    for recipe in pkg_recipes():
1499        packageFromRecipe(pkgcontents, recipe)
1500
1501    rsrcDir = os.path.join(pkgroot, 'Resources')
1502
1503    fn = os.path.join(pkgroot, 'PkgInfo')
1504    fp = open(fn, 'w')
1505    fp.write('pmkrpkg1')
1506    fp.close()
1507
1508    os.mkdir(rsrcDir)
1509
1510    makeMpkgPlist(os.path.join(pkgroot, 'Info.plist'))
1511    pl = Plist(
1512                IFPkgDescriptionTitle="Python",
1513                IFPkgDescriptionVersion=getVersion(),
1514            )
1515
1516    writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist'))
1517    for fn in os.listdir('resources'):
1518        if fn == '.svn': continue
1519        if fn.endswith('.jpg'):
1520            shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn))
1521        else:
1522            patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn))
1523
1524
1525def installSize(clear=False, _saved=[]):
1526    if clear:
1527        del _saved[:]
1528    if not _saved:
1529        data = captureCommand("du -ks %s"%(
1530                    shellQuote(os.path.join(WORKDIR, '_root'))))
1531        _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),))
1532    return _saved[0]
1533
1534
1535def buildDMG():
1536    """
1537    Create DMG containing the rootDir.
1538    """
1539    outdir = os.path.join(WORKDIR, 'diskimage')
1540    if os.path.exists(outdir):
1541        shutil.rmtree(outdir)
1542
1543    imagepath = os.path.join(outdir,
1544                    'python-%s-macosx%s'%(getFullVersion(),DEPTARGET))
1545    if INCLUDE_TIMESTAMP:
1546        imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3])
1547    imagepath = imagepath + '.dmg'
1548
1549    os.mkdir(outdir)
1550    volname='Python %s'%(getFullVersion())
1551    runCommand("hdiutil create -format UDRW -volname %s -srcfolder %s %s"%(
1552            shellQuote(volname),
1553            shellQuote(os.path.join(WORKDIR, 'installer')),
1554            shellQuote(imagepath + ".tmp.dmg" )))
1555
1556
1557    if not os.path.exists(os.path.join(WORKDIR, "mnt")):
1558        os.mkdir(os.path.join(WORKDIR, "mnt"))
1559    runCommand("hdiutil attach %s -mountroot %s"%(
1560        shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt"))))
1561
1562    # Custom icon for the DMG, shown when the DMG is mounted.
1563    shutil.copy("../Icons/Disk Image.icns",
1564            os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns"))
1565    runCommand("SetFile -a C %s/"%(
1566            shellQuote(os.path.join(WORKDIR, "mnt", volname)),))
1567
1568    runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname))))
1569
1570    setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns")
1571    runCommand("hdiutil convert %s -format UDZO -o %s"%(
1572            shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath)))
1573    setIcon(imagepath, "../Icons/Disk Image.icns")
1574
1575    os.unlink(imagepath + ".tmp.dmg")
1576
1577    return imagepath
1578
1579
1580def setIcon(filePath, icnsPath):
1581    """
1582    Set the custom icon for the specified file or directory.
1583    """
1584
1585    dirPath = os.path.normpath(os.path.dirname(__file__))
1586    toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon")
1587    if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime:
1588        # NOTE: The tool is created inside an .app bundle, otherwise it won't work due
1589        # to connections to the window server.
1590        appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS")
1591        if not os.path.exists(appPath):
1592            os.makedirs(appPath)
1593        runCommand("cc -o %s %s/seticon.m -framework Cocoa"%(
1594            shellQuote(toolPath), shellQuote(dirPath)))
1595
1596    runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath),
1597        shellQuote(filePath)))
1598
1599def main():
1600    # First parse options and check if we can perform our work
1601    parseOptions()
1602    checkEnvironment()
1603
1604    os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET
1605    os.environ['CC'] = CC
1606    os.environ['CXX'] = CXX
1607
1608    if os.path.exists(WORKDIR):
1609        shutil.rmtree(WORKDIR)
1610    os.mkdir(WORKDIR)
1611
1612    os.environ['LC_ALL'] = 'C'
1613
1614    # Then build third-party libraries such as sleepycat DB4.
1615    buildLibraries()
1616
1617    # Now build python itself
1618    buildPython()
1619
1620    # And then build the documentation
1621    # Remove the Deployment Target from the shell
1622    # environment, it's no longer needed and
1623    # an unexpected build target can cause problems
1624    # when Sphinx and its dependencies need to
1625    # be (re-)installed.
1626    del os.environ['MACOSX_DEPLOYMENT_TARGET']
1627    buildPythonDocs()
1628
1629
1630    # Prepare the applications folder
1631    folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%(
1632        getVersion(),))
1633    fn = os.path.join(folder, "License.rtf")
1634    patchFile("resources/License.rtf",  fn)
1635    fn = os.path.join(folder, "ReadMe.rtf")
1636    patchFile("resources/ReadMe.rtf",  fn)
1637    fn = os.path.join(folder, "Update Shell Profile.command")
1638    patchScript("scripts/postflight.patch-profile",  fn)
1639    os.chmod(folder, STAT_0o755)
1640    setIcon(folder, "../Icons/Python Folder.icns")
1641
1642    # Create the installer
1643    buildInstaller()
1644
1645    # And copy the readme into the directory containing the installer
1646    patchFile('resources/ReadMe.rtf',
1647                os.path.join(WORKDIR, 'installer', 'ReadMe.rtf'))
1648
1649    # Ditto for the license file.
1650    patchFile('resources/License.rtf',
1651                os.path.join(WORKDIR, 'installer', 'License.rtf'))
1652
1653    fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w')
1654    fp.write("# BUILD INFO\n")
1655    fp.write("# Date: %s\n" % time.ctime())
1656    fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos)
1657    fp.close()
1658
1659    # And copy it to a DMG
1660    buildDMG()
1661
1662if __name__ == "__main__":
1663    main()
1664