• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Autodetecting setup.py script for building the Python extensions
2
3import argparse
4import importlib._bootstrap
5import importlib.machinery
6import importlib.util
7import logging
8import os
9import re
10import sys
11import sysconfig
12import warnings
13from glob import glob, escape
14import _osx_support
15
16
17try:
18    import subprocess
19    del subprocess
20    SUBPROCESS_BOOTSTRAP = False
21except ImportError:
22    # Bootstrap Python: distutils.spawn uses subprocess to build C extensions,
23    # subprocess requires C extensions built by setup.py like _posixsubprocess.
24    #
25    # Use _bootsubprocess which only uses the os module.
26    #
27    # It is dropped from sys.modules as soon as all C extension modules
28    # are built.
29    import _bootsubprocess
30    sys.modules['subprocess'] = _bootsubprocess
31    del _bootsubprocess
32    SUBPROCESS_BOOTSTRAP = True
33
34
35with warnings.catch_warnings():
36    # bpo-41282 (PEP 632) deprecated distutils but setup.py still uses it
37    warnings.filterwarnings(
38        "ignore",
39        "The distutils package is deprecated",
40        DeprecationWarning
41    )
42    warnings.filterwarnings(
43        "ignore",
44        "The distutils.sysconfig module is deprecated, use sysconfig instead",
45        DeprecationWarning
46    )
47
48    from distutils.command.build_ext import build_ext
49    from distutils.command.build_scripts import build_scripts
50    from distutils.command.install import install
51    from distutils.command.install_lib import install_lib
52    from distutils.core import Extension, setup
53    from distutils.errors import CCompilerError, DistutilsError
54    from distutils.spawn import find_executable
55
56
57# Compile extensions used to test Python?
58TEST_EXTENSIONS = (sysconfig.get_config_var('TEST_MODULES') == 'yes')
59
60# This global variable is used to hold the list of modules to be disabled.
61DISABLED_MODULE_LIST = []
62
63# --list-module-names option used by Tools/scripts/generate_module_names.py
64LIST_MODULE_NAMES = False
65
66
67logging.basicConfig(format='%(message)s', level=logging.INFO)
68log = logging.getLogger('setup')
69
70
71def get_platform():
72    # Cross compiling
73    if "_PYTHON_HOST_PLATFORM" in os.environ:
74        return os.environ["_PYTHON_HOST_PLATFORM"]
75
76    # Get value of sys.platform
77    if sys.platform.startswith('osf1'):
78        return 'osf1'
79    return sys.platform
80
81
82CROSS_COMPILING = ("_PYTHON_HOST_PLATFORM" in os.environ)
83HOST_PLATFORM = get_platform()
84MS_WINDOWS = (HOST_PLATFORM == 'win32')
85CYGWIN = (HOST_PLATFORM == 'cygwin')
86MACOS = (HOST_PLATFORM == 'darwin')
87AIX = (HOST_PLATFORM.startswith('aix'))
88VXWORKS = ('vxworks' in HOST_PLATFORM)
89CC = os.environ.get("CC")
90if not CC:
91    CC = sysconfig.get_config_var("CC")
92
93
94SUMMARY = """
95Python is an interpreted, interactive, object-oriented programming
96language. It is often compared to Tcl, Perl, Scheme or Java.
97
98Python combines remarkable power with very clear syntax. It has
99modules, classes, exceptions, very high level dynamic data types, and
100dynamic typing. There are interfaces to many system calls and
101libraries, as well as to various windowing systems (X11, Motif, Tk,
102Mac, MFC). New built-in modules are easily written in C or C++. Python
103is also usable as an extension language for applications that need a
104programmable interface.
105
106The Python implementation is portable: it runs on many brands of UNIX,
107on Windows, DOS, Mac, Amiga... If your favorite system isn't
108listed here, it may still be supported, if there's a C compiler for
109it. Ask around on comp.lang.python -- or just try compiling Python
110yourself.
111"""
112
113CLASSIFIERS = """
114Development Status :: 6 - Mature
115License :: OSI Approved :: Python Software Foundation License
116Natural Language :: English
117Programming Language :: C
118Programming Language :: Python
119Topic :: Software Development
120"""
121
122
123def run_command(cmd):
124    status = os.system(cmd)
125    return os.waitstatus_to_exitcode(status)
126
127
128# Set common compiler and linker flags derived from the Makefile,
129# reserved for building the interpreter and the stdlib modules.
130# See bpo-21121 and bpo-35257
131def set_compiler_flags(compiler_flags, compiler_py_flags_nodist):
132    flags = sysconfig.get_config_var(compiler_flags)
133    py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist)
134    sysconfig.get_config_vars()[compiler_flags] = flags + ' ' + py_flags_nodist
135
136
137def add_dir_to_list(dirlist, dir):
138    """Add the directory 'dir' to the list 'dirlist' (after any relative
139    directories) if:
140
141    1) 'dir' is not already in 'dirlist'
142    2) 'dir' actually exists, and is a directory.
143    """
144    if dir is None or not os.path.isdir(dir) or dir in dirlist:
145        return
146    for i, path in enumerate(dirlist):
147        if not os.path.isabs(path):
148            dirlist.insert(i + 1, dir)
149            return
150    dirlist.insert(0, dir)
151
152
153def sysroot_paths(make_vars, subdirs):
154    """Get the paths of sysroot sub-directories.
155
156    * make_vars: a sequence of names of variables of the Makefile where
157      sysroot may be set.
158    * subdirs: a sequence of names of subdirectories used as the location for
159      headers or libraries.
160    """
161
162    dirs = []
163    for var_name in make_vars:
164        var = sysconfig.get_config_var(var_name)
165        if var is not None:
166            m = re.search(r'--sysroot=([^"]\S*|"[^"]+")', var)
167            if m is not None:
168                sysroot = m.group(1).strip('"')
169                for subdir in subdirs:
170                    if os.path.isabs(subdir):
171                        subdir = subdir[1:]
172                    path = os.path.join(sysroot, subdir)
173                    if os.path.isdir(path):
174                        dirs.append(path)
175                break
176    return dirs
177
178
179MACOS_SDK_ROOT = None
180MACOS_SDK_SPECIFIED = None
181
182def macosx_sdk_root():
183    """Return the directory of the current macOS SDK.
184
185    If no SDK was explicitly configured, call the compiler to find which
186    include files paths are being searched by default.  Use '/' if the
187    compiler is searching /usr/include (meaning system header files are
188    installed) or use the root of an SDK if that is being searched.
189    (The SDK may be supplied via Xcode or via the Command Line Tools).
190    The SDK paths used by Apple-supplied tool chains depend on the
191    setting of various variables; see the xcrun man page for more info.
192    Also sets MACOS_SDK_SPECIFIED for use by macosx_sdk_specified().
193    """
194    global MACOS_SDK_ROOT, MACOS_SDK_SPECIFIED
195
196    # If already called, return cached result.
197    if MACOS_SDK_ROOT:
198        return MACOS_SDK_ROOT
199
200    cflags = sysconfig.get_config_var('CFLAGS')
201    m = re.search(r'-isysroot\s*(\S+)', cflags)
202    if m is not None:
203        MACOS_SDK_ROOT = m.group(1)
204        MACOS_SDK_SPECIFIED = MACOS_SDK_ROOT != '/'
205    else:
206        MACOS_SDK_ROOT = _osx_support._default_sysroot(
207            sysconfig.get_config_var('CC'))
208        MACOS_SDK_SPECIFIED = False
209
210    return MACOS_SDK_ROOT
211
212
213def macosx_sdk_specified():
214    """Returns true if an SDK was explicitly configured.
215
216    True if an SDK was selected at configure time, either by specifying
217    --enable-universalsdk=(something other than no or /) or by adding a
218    -isysroot option to CFLAGS.  In some cases, like when making
219    decisions about macOS Tk framework paths, we need to be able to
220    know whether the user explicitly asked to build with an SDK versus
221    the implicit use of an SDK when header files are no longer
222    installed on a running system by the Command Line Tools.
223    """
224    global MACOS_SDK_SPECIFIED
225
226    # If already called, return cached result.
227    if MACOS_SDK_SPECIFIED:
228        return MACOS_SDK_SPECIFIED
229
230    # Find the sdk root and set MACOS_SDK_SPECIFIED
231    macosx_sdk_root()
232    return MACOS_SDK_SPECIFIED
233
234
235def is_macosx_sdk_path(path):
236    """
237    Returns True if 'path' can be located in a macOS SDK
238    """
239    return ( (path.startswith('/usr/') and not path.startswith('/usr/local'))
240                or path.startswith('/System/Library')
241                or path.startswith('/System/iOSSupport') )
242
243
244def grep_headers_for(function, headers):
245    for header in headers:
246        with open(header, 'r', errors='surrogateescape') as f:
247            if function in f.read():
248                return True
249    return False
250
251
252def find_file(filename, std_dirs, paths):
253    """Searches for the directory where a given file is located,
254    and returns a possibly-empty list of additional directories, or None
255    if the file couldn't be found at all.
256
257    'filename' is the name of a file, such as readline.h or libcrypto.a.
258    'std_dirs' is the list of standard system directories; if the
259        file is found in one of them, no additional directives are needed.
260    'paths' is a list of additional locations to check; if the file is
261        found in one of them, the resulting list will contain the directory.
262    """
263    if MACOS:
264        # Honor the MacOSX SDK setting when one was specified.
265        # An SDK is a directory with the same structure as a real
266        # system, but with only header files and libraries.
267        sysroot = macosx_sdk_root()
268
269    # Check the standard locations
270    for dir_ in std_dirs:
271        f = os.path.join(dir_, filename)
272
273        if MACOS and is_macosx_sdk_path(dir_):
274            f = os.path.join(sysroot, dir_[1:], filename)
275
276        if os.path.exists(f): return []
277
278    # Check the additional directories
279    for dir_ in paths:
280        f = os.path.join(dir_, filename)
281
282        if MACOS and is_macosx_sdk_path(dir_):
283            f = os.path.join(sysroot, dir_[1:], filename)
284
285        if os.path.exists(f):
286            return [dir_]
287
288    # Not found anywhere
289    return None
290
291
292def find_library_file(compiler, libname, std_dirs, paths):
293    result = compiler.find_library_file(std_dirs + paths, libname)
294    if result is None:
295        return None
296
297    if MACOS:
298        sysroot = macosx_sdk_root()
299
300    # Check whether the found file is in one of the standard directories
301    dirname = os.path.dirname(result)
302    for p in std_dirs:
303        # Ensure path doesn't end with path separator
304        p = p.rstrip(os.sep)
305
306        if MACOS and is_macosx_sdk_path(p):
307            # Note that, as of Xcode 7, Apple SDKs may contain textual stub
308            # libraries with .tbd extensions rather than the normal .dylib
309            # shared libraries installed in /.  The Apple compiler tool
310            # chain handles this transparently but it can cause problems
311            # for programs that are being built with an SDK and searching
312            # for specific libraries.  Distutils find_library_file() now
313            # knows to also search for and return .tbd files.  But callers
314            # of find_library_file need to keep in mind that the base filename
315            # of the returned SDK library file might have a different extension
316            # from that of the library file installed on the running system,
317            # for example:
318            #   /Applications/Xcode.app/Contents/Developer/Platforms/
319            #       MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/
320            #       usr/lib/libedit.tbd
321            # vs
322            #   /usr/lib/libedit.dylib
323            if os.path.join(sysroot, p[1:]) == dirname:
324                return [ ]
325
326        if p == dirname:
327            return [ ]
328
329    # Otherwise, it must have been in one of the additional directories,
330    # so we have to figure out which one.
331    for p in paths:
332        # Ensure path doesn't end with path separator
333        p = p.rstrip(os.sep)
334
335        if MACOS and is_macosx_sdk_path(p):
336            if os.path.join(sysroot, p[1:]) == dirname:
337                return [ p ]
338
339        if p == dirname:
340            return [p]
341    else:
342        assert False, "Internal error: Path not found in std_dirs or paths"
343
344
345def validate_tzpath():
346    base_tzpath = sysconfig.get_config_var('TZPATH')
347    if not base_tzpath:
348        return
349
350    tzpaths = base_tzpath.split(os.pathsep)
351    bad_paths = [tzpath for tzpath in tzpaths if not os.path.isabs(tzpath)]
352    if bad_paths:
353        raise ValueError('TZPATH must contain only absolute paths, '
354                         + f'found:\n{tzpaths!r}\nwith invalid paths:\n'
355                         + f'{bad_paths!r}')
356
357
358def find_module_file(module, dirlist):
359    """Find a module in a set of possible folders. If it is not found
360    return the unadorned filename"""
361    dirs = find_file(module, [], dirlist)
362    if not dirs:
363        return module
364    if len(dirs) > 1:
365        log.info(f"WARNING: multiple copies of {module} found")
366    return os.path.join(dirs[0], module)
367
368
369class PyBuildExt(build_ext):
370
371    def __init__(self, dist):
372        build_ext.__init__(self, dist)
373        self.srcdir = None
374        self.lib_dirs = None
375        self.inc_dirs = None
376        self.config_h_vars = None
377        self.failed = []
378        self.failed_on_import = []
379        self.missing = []
380        self.disabled_configure = []
381        if '-j' in os.environ.get('MAKEFLAGS', ''):
382            self.parallel = True
383
384    def add(self, ext):
385        self.extensions.append(ext)
386
387    def set_srcdir(self):
388        self.srcdir = sysconfig.get_config_var('srcdir')
389        if not self.srcdir:
390            # Maybe running on Windows but not using CYGWIN?
391            raise ValueError("No source directory; cannot proceed.")
392        self.srcdir = os.path.abspath(self.srcdir)
393
394    def remove_disabled(self):
395        # Remove modules that are present on the disabled list
396        extensions = [ext for ext in self.extensions
397                      if ext.name not in DISABLED_MODULE_LIST]
398        # move ctypes to the end, it depends on other modules
399        ext_map = dict((ext.name, i) for i, ext in enumerate(extensions))
400        if "_ctypes" in ext_map:
401            ctypes = extensions.pop(ext_map["_ctypes"])
402            extensions.append(ctypes)
403        self.extensions = extensions
404
405    def update_sources_depends(self):
406        # Fix up the autodetected modules, prefixing all the source files
407        # with Modules/.
408        moddirlist = [os.path.join(self.srcdir, 'Modules')]
409
410        # Fix up the paths for scripts, too
411        self.distribution.scripts = [os.path.join(self.srcdir, filename)
412                                     for filename in self.distribution.scripts]
413
414        # Python header files
415        headers = [sysconfig.get_config_h_filename()]
416        headers += glob(os.path.join(escape(sysconfig.get_path('include')), "*.h"))
417
418        for ext in self.extensions:
419            ext.sources = [ find_module_file(filename, moddirlist)
420                            for filename in ext.sources ]
421            if ext.depends is not None:
422                ext.depends = [find_module_file(filename, moddirlist)
423                               for filename in ext.depends]
424            else:
425                ext.depends = []
426            # re-compile extensions if a header file has been changed
427            ext.depends.extend(headers)
428
429    def remove_configured_extensions(self):
430        # The sysconfig variables built by makesetup that list the already
431        # built modules and the disabled modules as configured by the Setup
432        # files.
433        sysconf_built = sysconfig.get_config_var('MODBUILT_NAMES').split()
434        sysconf_dis = sysconfig.get_config_var('MODDISABLED_NAMES').split()
435
436        mods_built = []
437        mods_disabled = []
438        for ext in self.extensions:
439            # If a module has already been built or has been disabled in the
440            # Setup files, don't build it here.
441            if ext.name in sysconf_built:
442                mods_built.append(ext)
443            if ext.name in sysconf_dis:
444                mods_disabled.append(ext)
445
446        mods_configured = mods_built + mods_disabled
447        if mods_configured:
448            self.extensions = [x for x in self.extensions if x not in
449                               mods_configured]
450            # Remove the shared libraries built by a previous build.
451            for ext in mods_configured:
452                fullpath = self.get_ext_fullpath(ext.name)
453                if os.path.exists(fullpath):
454                    os.unlink(fullpath)
455
456        return (mods_built, mods_disabled)
457
458    def set_compiler_executables(self):
459        # When you run "make CC=altcc" or something similar, you really want
460        # those environment variables passed into the setup.py phase.  Here's
461        # a small set of useful ones.
462        compiler = os.environ.get('CC')
463        args = {}
464        # unfortunately, distutils doesn't let us provide separate C and C++
465        # compilers
466        if compiler is not None:
467            (ccshared,cflags) = sysconfig.get_config_vars('CCSHARED','CFLAGS')
468            args['compiler_so'] = compiler + ' ' + ccshared + ' ' + cflags
469        self.compiler.set_executables(**args)
470
471    def build_extensions(self):
472        self.set_srcdir()
473        self.set_compiler_executables()
474        self.configure_compiler()
475        self.init_inc_lib_dirs()
476
477        # Detect which modules should be compiled
478        self.detect_modules()
479
480        if not LIST_MODULE_NAMES:
481            self.remove_disabled()
482
483        self.update_sources_depends()
484        mods_built, mods_disabled = self.remove_configured_extensions()
485
486        if LIST_MODULE_NAMES:
487            for ext in self.extensions:
488                print(ext.name)
489            for name in self.missing:
490                print(name)
491            return
492
493        build_ext.build_extensions(self)
494
495        if SUBPROCESS_BOOTSTRAP:
496            # Drop our custom subprocess module:
497            # use the newly built subprocess module
498            del sys.modules['subprocess']
499
500        for ext in self.extensions:
501            self.check_extension_import(ext)
502
503        self.summary(mods_built, mods_disabled)
504
505    def summary(self, mods_built, mods_disabled):
506        longest = max([len(e.name) for e in self.extensions], default=0)
507        if self.failed or self.failed_on_import:
508            all_failed = self.failed + self.failed_on_import
509            longest = max(longest, max([len(name) for name in all_failed]))
510
511        def print_three_column(lst):
512            lst.sort(key=str.lower)
513            # guarantee zip() doesn't drop anything
514            while len(lst) % 3:
515                lst.append("")
516            for e, f, g in zip(lst[::3], lst[1::3], lst[2::3]):
517                print("%-*s   %-*s   %-*s" % (longest, e, longest, f,
518                                              longest, g))
519
520        if self.missing:
521            print()
522            print("Python build finished successfully!")
523            print("The necessary bits to build these optional modules were not "
524                  "found:")
525            print_three_column(self.missing)
526            print("To find the necessary bits, look in setup.py in"
527                  " detect_modules() for the module's name.")
528            print()
529
530        if mods_built:
531            print()
532            print("The following modules found by detect_modules() in"
533            " setup.py, have been")
534            print("built by the Makefile instead, as configured by the"
535            " Setup files:")
536            print_three_column([ext.name for ext in mods_built])
537            print()
538
539        if mods_disabled:
540            print()
541            print("The following modules found by detect_modules() in"
542            " setup.py have not")
543            print("been built, they are *disabled* in the Setup files:")
544            print_three_column([ext.name for ext in mods_disabled])
545            print()
546
547        if self.disabled_configure:
548            print()
549            print("The following modules found by detect_modules() in"
550            " setup.py have not")
551            print("been built, they are *disabled* by configure:")
552            print_three_column(self.disabled_configure)
553            print()
554
555        if self.failed:
556            failed = self.failed[:]
557            print()
558            print("Failed to build these modules:")
559            print_three_column(failed)
560            print()
561
562        if self.failed_on_import:
563            failed = self.failed_on_import[:]
564            print()
565            print("Following modules built successfully"
566                  " but were removed because they could not be imported:")
567            print_three_column(failed)
568            print()
569
570        if any('_ssl' in l
571               for l in (self.missing, self.failed, self.failed_on_import)):
572            print()
573            print("Could not build the ssl module!")
574            print("Python requires a OpenSSL 1.1.1 or newer")
575            if sysconfig.get_config_var("OPENSSL_LDFLAGS"):
576                print("Custom linker flags may require --with-openssl-rpath=auto")
577            print()
578
579        if os.environ.get("PYTHONSTRICTEXTENSIONBUILD") and (self.failed or self.failed_on_import):
580            raise RuntimeError("Failed to build some stdlib modules")
581
582    def build_extension(self, ext):
583
584        if ext.name == '_ctypes':
585            if not self.configure_ctypes(ext):
586                self.failed.append(ext.name)
587                return
588
589        try:
590            build_ext.build_extension(self, ext)
591        except (CCompilerError, DistutilsError) as why:
592            self.announce('WARNING: building of extension "%s" failed: %s' %
593                          (ext.name, why))
594            self.failed.append(ext.name)
595            return
596
597    def check_extension_import(self, ext):
598        # Don't try to import an extension that has failed to compile
599        if ext.name in self.failed:
600            self.announce(
601                'WARNING: skipping import check for failed build "%s"' %
602                ext.name, level=1)
603            return
604
605        # Workaround for Mac OS X: The Carbon-based modules cannot be
606        # reliably imported into a command-line Python
607        if 'Carbon' in ext.extra_link_args:
608            self.announce(
609                'WARNING: skipping import check for Carbon-based "%s"' %
610                ext.name)
611            return
612
613        if MACOS and (
614                sys.maxsize > 2**32 and '-arch' in ext.extra_link_args):
615            # Don't bother doing an import check when an extension was
616            # build with an explicit '-arch' flag on OSX. That's currently
617            # only used to build 32-bit only extensions in a 4-way
618            # universal build and loading 32-bit code into a 64-bit
619            # process will fail.
620            self.announce(
621                'WARNING: skipping import check for "%s"' %
622                ext.name)
623            return
624
625        # Workaround for Cygwin: Cygwin currently has fork issues when many
626        # modules have been imported
627        if CYGWIN:
628            self.announce('WARNING: skipping import check for Cygwin-based "%s"'
629                % ext.name)
630            return
631        ext_filename = os.path.join(
632            self.build_lib,
633            self.get_ext_filename(self.get_ext_fullname(ext.name)))
634
635        # If the build directory didn't exist when setup.py was
636        # started, sys.path_importer_cache has a negative result
637        # cached.  Clear that cache before trying to import.
638        sys.path_importer_cache.clear()
639
640        # Don't try to load extensions for cross builds
641        if CROSS_COMPILING:
642            return
643
644        loader = importlib.machinery.ExtensionFileLoader(ext.name, ext_filename)
645        spec = importlib.util.spec_from_file_location(ext.name, ext_filename,
646                                                      loader=loader)
647        try:
648            importlib._bootstrap._load(spec)
649        except ImportError as why:
650            self.failed_on_import.append(ext.name)
651            self.announce('*** WARNING: renaming "%s" since importing it'
652                          ' failed: %s' % (ext.name, why), level=3)
653            assert not self.inplace
654            basename, tail = os.path.splitext(ext_filename)
655            newname = basename + "_failed" + tail
656            if os.path.exists(newname):
657                os.remove(newname)
658            os.rename(ext_filename, newname)
659
660        except:
661            exc_type, why, tb = sys.exc_info()
662            self.announce('*** WARNING: importing extension "%s" '
663                          'failed with %s: %s' % (ext.name, exc_type, why),
664                          level=3)
665            self.failed.append(ext.name)
666
667    def add_multiarch_paths(self):
668        # Debian/Ubuntu multiarch support.
669        # https://wiki.ubuntu.com/MultiarchSpec
670        tmpfile = os.path.join(self.build_temp, 'multiarch')
671        if not os.path.exists(self.build_temp):
672            os.makedirs(self.build_temp)
673        ret = run_command(
674            '%s -print-multiarch > %s 2> /dev/null' % (CC, tmpfile))
675        multiarch_path_component = ''
676        try:
677            if ret == 0:
678                with open(tmpfile) as fp:
679                    multiarch_path_component = fp.readline().strip()
680        finally:
681            os.unlink(tmpfile)
682
683        if multiarch_path_component != '':
684            add_dir_to_list(self.compiler.library_dirs,
685                            '/usr/lib/' + multiarch_path_component)
686            add_dir_to_list(self.compiler.include_dirs,
687                            '/usr/include/' + multiarch_path_component)
688            return
689
690        if not find_executable('dpkg-architecture'):
691            return
692        opt = ''
693        if CROSS_COMPILING:
694            opt = '-t' + sysconfig.get_config_var('HOST_GNU_TYPE')
695        tmpfile = os.path.join(self.build_temp, 'multiarch')
696        if not os.path.exists(self.build_temp):
697            os.makedirs(self.build_temp)
698        ret = run_command(
699            'dpkg-architecture %s -qDEB_HOST_MULTIARCH > %s 2> /dev/null' %
700            (opt, tmpfile))
701        try:
702            if ret == 0:
703                with open(tmpfile) as fp:
704                    multiarch_path_component = fp.readline().strip()
705                add_dir_to_list(self.compiler.library_dirs,
706                                '/usr/lib/' + multiarch_path_component)
707                add_dir_to_list(self.compiler.include_dirs,
708                                '/usr/include/' + multiarch_path_component)
709        finally:
710            os.unlink(tmpfile)
711
712    def add_wrcc_search_dirs(self):
713        # add library search path by wr-cc, the compiler wrapper
714
715        def convert_mixed_path(path):
716            # convert path like C:\folder1\folder2/folder3/folder4
717            # to msys style /c/folder1/folder2/folder3/folder4
718            drive = path[0].lower()
719            left = path[2:].replace("\\", "/")
720            return "/" + drive + left
721
722        def add_search_path(line):
723            # On Windows building machine, VxWorks does
724            # cross builds under msys2 environment.
725            pathsep = (";" if sys.platform == "msys" else ":")
726            for d in line.strip().split("=")[1].split(pathsep):
727                d = d.strip()
728                if sys.platform == "msys":
729                    # On Windows building machine, compiler
730                    # returns mixed style path like:
731                    # C:\folder1\folder2/folder3/folder4
732                    d = convert_mixed_path(d)
733                d = os.path.normpath(d)
734                add_dir_to_list(self.compiler.library_dirs, d)
735
736        tmpfile = os.path.join(self.build_temp, 'wrccpaths')
737        os.makedirs(self.build_temp, exist_ok=True)
738        try:
739            ret = run_command('%s --print-search-dirs >%s' % (CC, tmpfile))
740            if ret:
741                return
742            with open(tmpfile) as fp:
743                # Parse paths in libraries line. The line is like:
744                # On Linux, "libraries: = path1:path2:path3"
745                # On Windows, "libraries: = path1;path2;path3"
746                for line in fp:
747                    if not line.startswith("libraries"):
748                        continue
749                    add_search_path(line)
750        finally:
751            try:
752                os.unlink(tmpfile)
753            except OSError:
754                pass
755
756    def add_cross_compiling_paths(self):
757        tmpfile = os.path.join(self.build_temp, 'ccpaths')
758        if not os.path.exists(self.build_temp):
759            os.makedirs(self.build_temp)
760        # bpo-38472: With a German locale, GCC returns "gcc-Version 9.1.0
761        # (GCC)", whereas it returns "gcc version 9.1.0" with the C locale.
762        ret = run_command('LC_ALL=C %s -E -v - </dev/null 2>%s 1>/dev/null' % (CC, tmpfile))
763        is_gcc = False
764        is_clang = False
765        in_incdirs = False
766        try:
767            if ret == 0:
768                with open(tmpfile) as fp:
769                    for line in fp.readlines():
770                        if line.startswith("gcc version"):
771                            is_gcc = True
772                        elif line.startswith("clang version"):
773                            is_clang = True
774                        elif line.startswith("#include <...>"):
775                            in_incdirs = True
776                        elif line.startswith("End of search list"):
777                            in_incdirs = False
778                        elif (is_gcc or is_clang) and line.startswith("LIBRARY_PATH"):
779                            for d in line.strip().split("=")[1].split(":"):
780                                d = os.path.normpath(d)
781                                if '/gcc/' not in d:
782                                    add_dir_to_list(self.compiler.library_dirs,
783                                                    d)
784                        elif (is_gcc or is_clang) and in_incdirs and '/gcc/' not in line and '/clang/' not in line:
785                            add_dir_to_list(self.compiler.include_dirs,
786                                            line.strip())
787        finally:
788            os.unlink(tmpfile)
789
790        if VXWORKS:
791            self.add_wrcc_search_dirs()
792
793    def add_ldflags_cppflags(self):
794        # Add paths specified in the environment variables LDFLAGS and
795        # CPPFLAGS for header and library files.
796        # We must get the values from the Makefile and not the environment
797        # directly since an inconsistently reproducible issue comes up where
798        # the environment variable is not set even though the value were passed
799        # into configure and stored in the Makefile (issue found on OS X 10.3).
800        for env_var, arg_name, dir_list in (
801                ('LDFLAGS', '-R', self.compiler.runtime_library_dirs),
802                ('LDFLAGS', '-L', self.compiler.library_dirs),
803                ('CPPFLAGS', '-I', self.compiler.include_dirs)):
804            env_val = sysconfig.get_config_var(env_var)
805            if env_val:
806                parser = argparse.ArgumentParser()
807                parser.add_argument(arg_name, dest="dirs", action="append")
808
809                # To prevent argparse from raising an exception about any
810                # options in env_val that it mistakes for known option, we
811                # strip out all double dashes and any dashes followed by a
812                # character that is not for the option we are dealing with.
813                #
814                # Please note that order of the regex is important!  We must
815                # strip out double-dashes first so that we don't end up with
816                # substituting "--Long" to "-Long" and thus lead to "ong" being
817                # used for a library directory.
818                env_val = re.sub(r'(^|\s+)-(-|(?!%s))' % arg_name[1],
819                                 ' ', env_val)
820                options, _ = parser.parse_known_args(env_val.split())
821                if options.dirs:
822                    for directory in reversed(options.dirs):
823                        add_dir_to_list(dir_list, directory)
824
825    def configure_compiler(self):
826        # Ensure that /usr/local is always used, but the local build
827        # directories (i.e. '.' and 'Include') must be first.  See issue
828        # 10520.
829        if not CROSS_COMPILING:
830            add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib')
831            add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
832        # only change this for cross builds for 3.3, issues on Mageia
833        if CROSS_COMPILING:
834            self.add_cross_compiling_paths()
835        self.add_multiarch_paths()
836        self.add_ldflags_cppflags()
837
838    def init_inc_lib_dirs(self):
839        if (not CROSS_COMPILING and
840                os.path.normpath(sys.base_prefix) != '/usr' and
841                not sysconfig.get_config_var('PYTHONFRAMEWORK')):
842            # OSX note: Don't add LIBDIR and INCLUDEDIR to building a framework
843            # (PYTHONFRAMEWORK is set) to avoid # linking problems when
844            # building a framework with different architectures than
845            # the one that is currently installed (issue #7473)
846            add_dir_to_list(self.compiler.library_dirs,
847                            sysconfig.get_config_var("LIBDIR"))
848            add_dir_to_list(self.compiler.include_dirs,
849                            sysconfig.get_config_var("INCLUDEDIR"))
850
851        system_lib_dirs = ['/lib64', '/usr/lib64', '/lib', '/usr/lib']
852        system_include_dirs = ['/usr/include']
853        # lib_dirs and inc_dirs are used to search for files;
854        # if a file is found in one of those directories, it can
855        # be assumed that no additional -I,-L directives are needed.
856        if not CROSS_COMPILING:
857            self.lib_dirs = self.compiler.library_dirs + system_lib_dirs
858            self.inc_dirs = self.compiler.include_dirs + system_include_dirs
859        else:
860            # Add the sysroot paths. 'sysroot' is a compiler option used to
861            # set the logical path of the standard system headers and
862            # libraries.
863            self.lib_dirs = (self.compiler.library_dirs +
864                             sysroot_paths(('LDFLAGS', 'CC'), system_lib_dirs))
865            self.inc_dirs = (self.compiler.include_dirs +
866                             sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'),
867                                           system_include_dirs))
868
869        config_h = sysconfig.get_config_h_filename()
870        with open(config_h) as file:
871            self.config_h_vars = sysconfig.parse_config_h(file)
872
873        # OSF/1 and Unixware have some stuff in /usr/ccs/lib (like -ldb)
874        if HOST_PLATFORM in ['osf1', 'unixware7', 'openunix8']:
875            self.lib_dirs += ['/usr/ccs/lib']
876
877        # HP-UX11iv3 keeps files in lib/hpux folders.
878        if HOST_PLATFORM == 'hp-ux11':
879            self.lib_dirs += ['/usr/lib/hpux64', '/usr/lib/hpux32']
880
881        if MACOS:
882            # This should work on any unixy platform ;-)
883            # If the user has bothered specifying additional -I and -L flags
884            # in OPT and LDFLAGS we might as well use them here.
885            #
886            # NOTE: using shlex.split would technically be more correct, but
887            # also gives a bootstrap problem. Let's hope nobody uses
888            # directories with whitespace in the name to store libraries.
889            cflags, ldflags = sysconfig.get_config_vars(
890                    'CFLAGS', 'LDFLAGS')
891            for item in cflags.split():
892                if item.startswith('-I'):
893                    self.inc_dirs.append(item[2:])
894
895            for item in ldflags.split():
896                if item.startswith('-L'):
897                    self.lib_dirs.append(item[2:])
898
899    def detect_simple_extensions(self):
900        #
901        # The following modules are all pretty straightforward, and compile
902        # on pretty much any POSIXish platform.
903        #
904
905        # array objects
906        self.add(Extension('array', ['arraymodule.c'],
907                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
908
909        # Context Variables
910        self.add(Extension('_contextvars', ['_contextvarsmodule.c']))
911
912        shared_math = 'Modules/_math.o'
913
914        # math library functions, e.g. sin()
915        self.add(Extension('math',  ['mathmodule.c'],
916                           extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
917                           extra_objects=[shared_math],
918                           depends=['_math.h', shared_math],
919                           libraries=['m']))
920
921        # complex math library functions
922        self.add(Extension('cmath', ['cmathmodule.c'],
923                           extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
924                           extra_objects=[shared_math],
925                           depends=['_math.h', shared_math],
926                           libraries=['m']))
927
928        # time libraries: librt may be needed for clock_gettime()
929        time_libs = []
930        lib = sysconfig.get_config_var('TIMEMODULE_LIB')
931        if lib:
932            time_libs.append(lib)
933
934        # time operations and variables
935        self.add(Extension('time', ['timemodule.c'],
936                           libraries=time_libs))
937        # libm is needed by delta_new() that uses round() and by accum() that
938        # uses modf().
939        self.add(Extension('_datetime', ['_datetimemodule.c'],
940                           libraries=['m'],
941                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
942        # zoneinfo module
943        self.add(Extension('_zoneinfo', ['_zoneinfo.c'],
944                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
945        # random number generator implemented in C
946        self.add(Extension("_random", ["_randommodule.c"],
947                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
948        # bisect
949        self.add(Extension("_bisect", ["_bisectmodule.c"]))
950        # heapq
951        self.add(Extension("_heapq", ["_heapqmodule.c"],
952                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
953        # C-optimized pickle replacement
954        self.add(Extension("_pickle", ["_pickle.c"],
955                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
956        # _json speedups
957        self.add(Extension("_json", ["_json.c"],
958                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
959
960        # profiler (_lsprof is for cProfile.py)
961        self.add(Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c']))
962        # static Unicode character database
963        self.add(Extension('unicodedata', ['unicodedata.c'],
964                           depends=['unicodedata_db.h', 'unicodename_db.h'],
965                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
966        # _opcode module
967        self.add(Extension('_opcode', ['_opcode.c']))
968        # asyncio speedups
969        self.add(Extension("_asyncio", ["_asynciomodule.c"],
970                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
971        # _abc speedups
972        self.add(Extension("_abc", ["_abc.c"],
973                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
974        # _queue module
975        self.add(Extension("_queue", ["_queuemodule.c"],
976                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
977        # _statistics module
978        self.add(Extension("_statistics", ["_statisticsmodule.c"]))
979
980        # Modules with some UNIX dependencies -- on by default:
981        # (If you have a really backward UNIX, select and socket may not be
982        # supported...)
983
984        # fcntl(2) and ioctl(2)
985        libs = []
986        if (self.config_h_vars.get('FLOCK_NEEDS_LIBBSD', False)):
987            # May be necessary on AIX for flock function
988            libs = ['bsd']
989        self.add(Extension('fcntl', ['fcntlmodule.c'],
990                           libraries=libs))
991        # pwd(3)
992        self.add(Extension('pwd', ['pwdmodule.c']))
993        # grp(3)
994        if not VXWORKS:
995            self.add(Extension('grp', ['grpmodule.c']))
996        # spwd, shadow passwords
997        if (self.config_h_vars.get('HAVE_GETSPNAM', False) or
998                self.config_h_vars.get('HAVE_GETSPENT', False)):
999            self.add(Extension('spwd', ['spwdmodule.c']))
1000        # AIX has shadow passwords, but access is not via getspent(), etc.
1001        # module support is not expected so it not 'missing'
1002        elif not AIX:
1003            self.missing.append('spwd')
1004
1005        # select(2); not on ancient System V
1006        self.add(Extension('select', ['selectmodule.c']))
1007
1008        # Memory-mapped files (also works on Win32).
1009        self.add(Extension('mmap', ['mmapmodule.c']))
1010
1011        # Lance Ellinghaus's syslog module
1012        # syslog daemon interface
1013        self.add(Extension('syslog', ['syslogmodule.c']))
1014
1015        # Python interface to subinterpreter C-API.
1016        self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c']))
1017
1018        #
1019        # Here ends the simple stuff.  From here on, modules need certain
1020        # libraries, are platform-specific, or present other surprises.
1021        #
1022
1023        # Multimedia modules
1024        # These don't work for 64-bit platforms!!!
1025        # These represent audio samples or images as strings:
1026        #
1027        # Operations on audio samples
1028        # According to #993173, this one should actually work fine on
1029        # 64-bit platforms.
1030        #
1031        # audioop needs libm for floor() in multiple functions.
1032        self.add(Extension('audioop', ['audioop.c'],
1033                           libraries=['m']))
1034
1035        # CSV files
1036        self.add(Extension('_csv', ['_csv.c']))
1037
1038        # POSIX subprocess module helper.
1039        self.add(Extension('_posixsubprocess', ['_posixsubprocess.c'],
1040                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
1041
1042    def detect_test_extensions(self):
1043        # Python C API test module
1044        self.add(Extension('_testcapi', ['_testcapimodule.c'],
1045                           depends=['testcapi_long.h']))
1046
1047        # Python Internal C API test module
1048        self.add(Extension('_testinternalcapi', ['_testinternalcapi.c'],
1049                           extra_compile_args=['-DPy_BUILD_CORE_MODULE']))
1050
1051        # Python PEP-3118 (buffer protocol) test module
1052        self.add(Extension('_testbuffer', ['_testbuffer.c']))
1053
1054        # Test loading multiple modules from one compiled file (https://bugs.python.org/issue16421)
1055        self.add(Extension('_testimportmultiple', ['_testimportmultiple.c']))
1056
1057        # Test multi-phase extension module init (PEP 489)
1058        self.add(Extension('_testmultiphase', ['_testmultiphase.c']))
1059
1060        # Fuzz tests.
1061        self.add(Extension('_xxtestfuzz',
1062                           ['_xxtestfuzz/_xxtestfuzz.c',
1063                            '_xxtestfuzz/fuzzer.c']))
1064
1065    def detect_readline_curses(self):
1066        # readline
1067        readline_termcap_library = ""
1068        curses_library = ""
1069        # Cannot use os.popen here in py3k.
1070        tmpfile = os.path.join(self.build_temp, 'readline_termcap_lib')
1071        if not os.path.exists(self.build_temp):
1072            os.makedirs(self.build_temp)
1073        # Determine if readline is already linked against curses or tinfo.
1074        if sysconfig.get_config_var('HAVE_LIBREADLINE'):
1075            if sysconfig.get_config_var('WITH_EDITLINE'):
1076                readline_lib = 'edit'
1077            else:
1078                readline_lib = 'readline'
1079            do_readline = self.compiler.find_library_file(self.lib_dirs,
1080                readline_lib)
1081            if CROSS_COMPILING:
1082                ret = run_command("%s -d %s | grep '(NEEDED)' > %s"
1083                                % (sysconfig.get_config_var('READELF'),
1084                                   do_readline, tmpfile))
1085            elif find_executable('ldd'):
1086                ret = run_command("ldd %s > %s" % (do_readline, tmpfile))
1087            else:
1088                ret = 1
1089            if ret == 0:
1090                with open(tmpfile) as fp:
1091                    for ln in fp:
1092                        if 'curses' in ln:
1093                            readline_termcap_library = re.sub(
1094                                r'.*lib(n?cursesw?)\.so.*', r'\1', ln
1095                            ).rstrip()
1096                            break
1097                        # termcap interface split out from ncurses
1098                        if 'tinfo' in ln:
1099                            readline_termcap_library = 'tinfo'
1100                            break
1101            if os.path.exists(tmpfile):
1102                os.unlink(tmpfile)
1103        else:
1104            do_readline = False
1105        # Issue 7384: If readline is already linked against curses,
1106        # use the same library for the readline and curses modules.
1107        if 'curses' in readline_termcap_library:
1108            curses_library = readline_termcap_library
1109        elif self.compiler.find_library_file(self.lib_dirs, 'ncursesw'):
1110            curses_library = 'ncursesw'
1111        # Issue 36210: OSS provided ncurses does not link on AIX
1112        # Use IBM supplied 'curses' for successful build of _curses
1113        elif AIX and self.compiler.find_library_file(self.lib_dirs, 'curses'):
1114            curses_library = 'curses'
1115        elif self.compiler.find_library_file(self.lib_dirs, 'ncurses'):
1116            curses_library = 'ncurses'
1117        elif self.compiler.find_library_file(self.lib_dirs, 'curses'):
1118            curses_library = 'curses'
1119
1120        if MACOS:
1121            os_release = int(os.uname()[2].split('.')[0])
1122            dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
1123            if (dep_target and
1124                    (tuple(int(n) for n in dep_target.split('.')[0:2])
1125                        < (10, 5) ) ):
1126                os_release = 8
1127            if os_release < 9:
1128                # MacOSX 10.4 has a broken readline. Don't try to build
1129                # the readline module unless the user has installed a fixed
1130                # readline package
1131                if find_file('readline/rlconf.h', self.inc_dirs, []) is None:
1132                    do_readline = False
1133        if do_readline:
1134            if MACOS and os_release < 9:
1135                # In every directory on the search path search for a dynamic
1136                # library and then a static library, instead of first looking
1137                # for dynamic libraries on the entire path.
1138                # This way a statically linked custom readline gets picked up
1139                # before the (possibly broken) dynamic library in /usr/lib.
1140                readline_extra_link_args = ('-Wl,-search_paths_first',)
1141            else:
1142                readline_extra_link_args = ()
1143
1144            readline_libs = [readline_lib]
1145            if readline_termcap_library:
1146                pass # Issue 7384: Already linked against curses or tinfo.
1147            elif curses_library:
1148                readline_libs.append(curses_library)
1149            elif self.compiler.find_library_file(self.lib_dirs +
1150                                                     ['/usr/lib/termcap'],
1151                                                     'termcap'):
1152                readline_libs.append('termcap')
1153            self.add(Extension('readline', ['readline.c'],
1154                               library_dirs=['/usr/lib/termcap'],
1155                               extra_link_args=readline_extra_link_args,
1156                               libraries=readline_libs))
1157        else:
1158            self.missing.append('readline')
1159
1160        # Curses support, requiring the System V version of curses, often
1161        # provided by the ncurses library.
1162        curses_defines = []
1163        curses_includes = []
1164        panel_library = 'panel'
1165        if curses_library == 'ncursesw':
1166            curses_defines.append(('HAVE_NCURSESW', '1'))
1167            if not CROSS_COMPILING:
1168                curses_includes.append('/usr/include/ncursesw')
1169            # Bug 1464056: If _curses.so links with ncursesw,
1170            # _curses_panel.so must link with panelw.
1171            panel_library = 'panelw'
1172            if MACOS:
1173                # On OS X, there is no separate /usr/lib/libncursesw nor
1174                # libpanelw.  If we are here, we found a locally-supplied
1175                # version of libncursesw.  There should also be a
1176                # libpanelw.  _XOPEN_SOURCE defines are usually excluded
1177                # for OS X but we need _XOPEN_SOURCE_EXTENDED here for
1178                # ncurses wide char support
1179                curses_defines.append(('_XOPEN_SOURCE_EXTENDED', '1'))
1180        elif MACOS and curses_library == 'ncurses':
1181            # Building with the system-suppied combined libncurses/libpanel
1182            curses_defines.append(('HAVE_NCURSESW', '1'))
1183            curses_defines.append(('_XOPEN_SOURCE_EXTENDED', '1'))
1184
1185        curses_enabled = True
1186        if curses_library.startswith('ncurses'):
1187            curses_libs = [curses_library]
1188            self.add(Extension('_curses', ['_cursesmodule.c'],
1189                               extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
1190                               include_dirs=curses_includes,
1191                               define_macros=curses_defines,
1192                               libraries=curses_libs))
1193        elif curses_library == 'curses' and not MACOS:
1194                # OSX has an old Berkeley curses, not good enough for
1195                # the _curses module.
1196            if (self.compiler.find_library_file(self.lib_dirs, 'terminfo')):
1197                curses_libs = ['curses', 'terminfo']
1198            elif (self.compiler.find_library_file(self.lib_dirs, 'termcap')):
1199                curses_libs = ['curses', 'termcap']
1200            else:
1201                curses_libs = ['curses']
1202
1203            self.add(Extension('_curses', ['_cursesmodule.c'],
1204                               extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
1205                               define_macros=curses_defines,
1206                               libraries=curses_libs))
1207        else:
1208            curses_enabled = False
1209            self.missing.append('_curses')
1210
1211        # If the curses module is enabled, check for the panel module
1212        # _curses_panel needs some form of ncurses
1213        skip_curses_panel = True if AIX else False
1214        if (curses_enabled and not skip_curses_panel and
1215                self.compiler.find_library_file(self.lib_dirs, panel_library)):
1216            self.add(Extension('_curses_panel', ['_curses_panel.c'],
1217                           include_dirs=curses_includes,
1218                           define_macros=curses_defines,
1219                           libraries=[panel_library, *curses_libs]))
1220        elif not skip_curses_panel:
1221            self.missing.append('_curses_panel')
1222
1223    def detect_crypt(self):
1224        # crypt module.
1225        if VXWORKS:
1226            # bpo-31904: crypt() function is not provided by VxWorks.
1227            # DES_crypt() OpenSSL provides is too weak to implement
1228            # the encryption.
1229            self.missing.append('_crypt')
1230            return
1231
1232        if self.compiler.find_library_file(self.lib_dirs, 'crypt'):
1233            libs = ['crypt']
1234        else:
1235            libs = []
1236
1237        self.add(Extension('_crypt', ['_cryptmodule.c'], libraries=libs))
1238
1239    def detect_socket(self):
1240        # socket(2)
1241        kwargs = {'depends': ['socketmodule.h']}
1242        if MACOS:
1243            # Issue #35569: Expose RFC 3542 socket options.
1244            kwargs['extra_compile_args'] = ['-D__APPLE_USE_RFC_3542']
1245
1246        self.add(Extension('_socket', ['socketmodule.c'], **kwargs))
1247
1248    def detect_dbm_gdbm(self):
1249        # Modules that provide persistent dictionary-like semantics.  You will
1250        # probably want to arrange for at least one of them to be available on
1251        # your machine, though none are defined by default because of library
1252        # dependencies.  The Python module dbm/__init__.py provides an
1253        # implementation independent wrapper for these; dbm/dumb.py provides
1254        # similar functionality (but slower of course) implemented in Python.
1255
1256        # Sleepycat^WOracle Berkeley DB interface.
1257        #  https://www.oracle.com/database/technologies/related/berkeleydb.html
1258        #
1259        # This requires the Sleepycat^WOracle DB code. The supported versions
1260        # are set below.  Visit the URL above to download
1261        # a release.  Most open source OSes come with one or more
1262        # versions of BerkeleyDB already installed.
1263
1264        max_db_ver = (5, 3)
1265        min_db_ver = (3, 3)
1266        db_setup_debug = False   # verbose debug prints from this script?
1267
1268        def allow_db_ver(db_ver):
1269            """Returns a boolean if the given BerkeleyDB version is acceptable.
1270
1271            Args:
1272              db_ver: A tuple of the version to verify.
1273            """
1274            if not (min_db_ver <= db_ver <= max_db_ver):
1275                return False
1276            return True
1277
1278        def gen_db_minor_ver_nums(major):
1279            if major == 4:
1280                for x in range(max_db_ver[1]+1):
1281                    if allow_db_ver((4, x)):
1282                        yield x
1283            elif major == 3:
1284                for x in (3,):
1285                    if allow_db_ver((3, x)):
1286                        yield x
1287            else:
1288                raise ValueError("unknown major BerkeleyDB version", major)
1289
1290        # construct a list of paths to look for the header file in on
1291        # top of the normal inc_dirs.
1292        db_inc_paths = [
1293            '/usr/include/db4',
1294            '/usr/local/include/db4',
1295            '/opt/sfw/include/db4',
1296            '/usr/include/db3',
1297            '/usr/local/include/db3',
1298            '/opt/sfw/include/db3',
1299            # Fink defaults (https://www.finkproject.org/)
1300            '/sw/include/db4',
1301            '/sw/include/db3',
1302        ]
1303        # 4.x minor number specific paths
1304        for x in gen_db_minor_ver_nums(4):
1305            db_inc_paths.append('/usr/include/db4%d' % x)
1306            db_inc_paths.append('/usr/include/db4.%d' % x)
1307            db_inc_paths.append('/usr/local/BerkeleyDB.4.%d/include' % x)
1308            db_inc_paths.append('/usr/local/include/db4%d' % x)
1309            db_inc_paths.append('/pkg/db-4.%d/include' % x)
1310            db_inc_paths.append('/opt/db-4.%d/include' % x)
1311            # MacPorts default (https://www.macports.org/)
1312            db_inc_paths.append('/opt/local/include/db4%d' % x)
1313        # 3.x minor number specific paths
1314        for x in gen_db_minor_ver_nums(3):
1315            db_inc_paths.append('/usr/include/db3%d' % x)
1316            db_inc_paths.append('/usr/local/BerkeleyDB.3.%d/include' % x)
1317            db_inc_paths.append('/usr/local/include/db3%d' % x)
1318            db_inc_paths.append('/pkg/db-3.%d/include' % x)
1319            db_inc_paths.append('/opt/db-3.%d/include' % x)
1320
1321        if CROSS_COMPILING:
1322            db_inc_paths = []
1323
1324        # Add some common subdirectories for Sleepycat DB to the list,
1325        # based on the standard include directories. This way DB3/4 gets
1326        # picked up when it is installed in a non-standard prefix and
1327        # the user has added that prefix into inc_dirs.
1328        std_variants = []
1329        for dn in self.inc_dirs:
1330            std_variants.append(os.path.join(dn, 'db3'))
1331            std_variants.append(os.path.join(dn, 'db4'))
1332            for x in gen_db_minor_ver_nums(4):
1333                std_variants.append(os.path.join(dn, "db4%d"%x))
1334                std_variants.append(os.path.join(dn, "db4.%d"%x))
1335            for x in gen_db_minor_ver_nums(3):
1336                std_variants.append(os.path.join(dn, "db3%d"%x))
1337                std_variants.append(os.path.join(dn, "db3.%d"%x))
1338
1339        db_inc_paths = std_variants + db_inc_paths
1340        db_inc_paths = [p for p in db_inc_paths if os.path.exists(p)]
1341
1342        db_ver_inc_map = {}
1343
1344        if MACOS:
1345            sysroot = macosx_sdk_root()
1346
1347        class db_found(Exception): pass
1348        try:
1349            # See whether there is a Sleepycat header in the standard
1350            # search path.
1351            for d in self.inc_dirs + db_inc_paths:
1352                f = os.path.join(d, "db.h")
1353                if MACOS and is_macosx_sdk_path(d):
1354                    f = os.path.join(sysroot, d[1:], "db.h")
1355
1356                if db_setup_debug: print("db: looking for db.h in", f)
1357                if os.path.exists(f):
1358                    with open(f, 'rb') as file:
1359                        f = file.read()
1360                    m = re.search(br"#define\WDB_VERSION_MAJOR\W(\d+)", f)
1361                    if m:
1362                        db_major = int(m.group(1))
1363                        m = re.search(br"#define\WDB_VERSION_MINOR\W(\d+)", f)
1364                        db_minor = int(m.group(1))
1365                        db_ver = (db_major, db_minor)
1366
1367                        # Avoid 4.6 prior to 4.6.21 due to a BerkeleyDB bug
1368                        if db_ver == (4, 6):
1369                            m = re.search(br"#define\WDB_VERSION_PATCH\W(\d+)", f)
1370                            db_patch = int(m.group(1))
1371                            if db_patch < 21:
1372                                print("db.h:", db_ver, "patch", db_patch,
1373                                      "being ignored (4.6.x must be >= 4.6.21)")
1374                                continue
1375
1376                        if ( (db_ver not in db_ver_inc_map) and
1377                            allow_db_ver(db_ver) ):
1378                            # save the include directory with the db.h version
1379                            # (first occurrence only)
1380                            db_ver_inc_map[db_ver] = d
1381                            if db_setup_debug:
1382                                print("db.h: found", db_ver, "in", d)
1383                        else:
1384                            # we already found a header for this library version
1385                            if db_setup_debug: print("db.h: ignoring", d)
1386                    else:
1387                        # ignore this header, it didn't contain a version number
1388                        if db_setup_debug:
1389                            print("db.h: no version number version in", d)
1390
1391            db_found_vers = list(db_ver_inc_map.keys())
1392            db_found_vers.sort()
1393
1394            while db_found_vers:
1395                db_ver = db_found_vers.pop()
1396                db_incdir = db_ver_inc_map[db_ver]
1397
1398                # check lib directories parallel to the location of the header
1399                db_dirs_to_check = [
1400                    db_incdir.replace("include", 'lib64'),
1401                    db_incdir.replace("include", 'lib'),
1402                ]
1403
1404                if not MACOS:
1405                    db_dirs_to_check = list(filter(os.path.isdir, db_dirs_to_check))
1406
1407                else:
1408                    # Same as other branch, but takes OSX SDK into account
1409                    tmp = []
1410                    for dn in db_dirs_to_check:
1411                        if is_macosx_sdk_path(dn):
1412                            if os.path.isdir(os.path.join(sysroot, dn[1:])):
1413                                tmp.append(dn)
1414                        else:
1415                            if os.path.isdir(dn):
1416                                tmp.append(dn)
1417                    db_dirs_to_check = tmp
1418
1419                    db_dirs_to_check = tmp
1420
1421                # Look for a version specific db-X.Y before an ambiguous dbX
1422                # XXX should we -ever- look for a dbX name?  Do any
1423                # systems really not name their library by version and
1424                # symlink to more general names?
1425                for dblib in (('db-%d.%d' % db_ver),
1426                              ('db%d%d' % db_ver),
1427                              ('db%d' % db_ver[0])):
1428                    dblib_file = self.compiler.find_library_file(
1429                                    db_dirs_to_check + self.lib_dirs, dblib )
1430                    if dblib_file:
1431                        dblib_dir = [ os.path.abspath(os.path.dirname(dblib_file)) ]
1432                        raise db_found
1433                    else:
1434                        if db_setup_debug: print("db lib: ", dblib, "not found")
1435
1436        except db_found:
1437            if db_setup_debug:
1438                print("bsddb using BerkeleyDB lib:", db_ver, dblib)
1439                print("bsddb lib dir:", dblib_dir, " inc dir:", db_incdir)
1440            dblibs = [dblib]
1441            # Only add the found library and include directories if they aren't
1442            # already being searched. This avoids an explicit runtime library
1443            # dependency.
1444            if db_incdir in self.inc_dirs:
1445                db_incs = None
1446            else:
1447                db_incs = [db_incdir]
1448            if dblib_dir[0] in self.lib_dirs:
1449                dblib_dir = None
1450        else:
1451            if db_setup_debug: print("db: no appropriate library found")
1452            db_incs = None
1453            dblibs = []
1454            dblib_dir = None
1455
1456        dbm_setup_debug = False   # verbose debug prints from this script?
1457        dbm_order = ['gdbm']
1458        # The standard Unix dbm module:
1459        if not CYGWIN:
1460            config_args = [arg.strip("'")
1461                           for arg in sysconfig.get_config_var("CONFIG_ARGS").split()]
1462            dbm_args = [arg for arg in config_args
1463                        if arg.startswith('--with-dbmliborder=')]
1464            if dbm_args:
1465                dbm_order = [arg.split('=')[-1] for arg in dbm_args][-1].split(":")
1466            else:
1467                dbm_order = "ndbm:gdbm:bdb".split(":")
1468            dbmext = None
1469            for cand in dbm_order:
1470                if cand == "ndbm":
1471                    if find_file("ndbm.h", self.inc_dirs, []) is not None:
1472                        # Some systems have -lndbm, others have -lgdbm_compat,
1473                        # others don't have either
1474                        if self.compiler.find_library_file(self.lib_dirs,
1475                                                               'ndbm'):
1476                            ndbm_libs = ['ndbm']
1477                        elif self.compiler.find_library_file(self.lib_dirs,
1478                                                             'gdbm_compat'):
1479                            ndbm_libs = ['gdbm_compat']
1480                        else:
1481                            ndbm_libs = []
1482                        if dbm_setup_debug: print("building dbm using ndbm")
1483                        dbmext = Extension('_dbm', ['_dbmmodule.c'],
1484                                           define_macros=[
1485                                               ('HAVE_NDBM_H',None),
1486                                               ],
1487                                           libraries=ndbm_libs)
1488                        break
1489
1490                elif cand == "gdbm":
1491                    if self.compiler.find_library_file(self.lib_dirs, 'gdbm'):
1492                        gdbm_libs = ['gdbm']
1493                        if self.compiler.find_library_file(self.lib_dirs,
1494                                                               'gdbm_compat'):
1495                            gdbm_libs.append('gdbm_compat')
1496                        if find_file("gdbm/ndbm.h", self.inc_dirs, []) is not None:
1497                            if dbm_setup_debug: print("building dbm using gdbm")
1498                            dbmext = Extension(
1499                                '_dbm', ['_dbmmodule.c'],
1500                                define_macros=[
1501                                    ('HAVE_GDBM_NDBM_H', None),
1502                                    ],
1503                                libraries = gdbm_libs)
1504                            break
1505                        if find_file("gdbm-ndbm.h", self.inc_dirs, []) is not None:
1506                            if dbm_setup_debug: print("building dbm using gdbm")
1507                            dbmext = Extension(
1508                                '_dbm', ['_dbmmodule.c'],
1509                                define_macros=[
1510                                    ('HAVE_GDBM_DASH_NDBM_H', None),
1511                                    ],
1512                                libraries = gdbm_libs)
1513                            break
1514                elif cand == "bdb":
1515                    if dblibs:
1516                        if dbm_setup_debug: print("building dbm using bdb")
1517                        dbmext = Extension('_dbm', ['_dbmmodule.c'],
1518                                           library_dirs=dblib_dir,
1519                                           runtime_library_dirs=dblib_dir,
1520                                           include_dirs=db_incs,
1521                                           define_macros=[
1522                                               ('HAVE_BERKDB_H', None),
1523                                               ('DB_DBM_HSEARCH', None),
1524                                               ],
1525                                           libraries=dblibs)
1526                        break
1527            if dbmext is not None:
1528                self.add(dbmext)
1529            else:
1530                self.missing.append('_dbm')
1531
1532        # Anthony Baxter's gdbm module.  GNU dbm(3) will require -lgdbm:
1533        if ('gdbm' in dbm_order and
1534            self.compiler.find_library_file(self.lib_dirs, 'gdbm')):
1535            self.add(Extension('_gdbm', ['_gdbmmodule.c'],
1536                               libraries=['gdbm']))
1537        else:
1538            self.missing.append('_gdbm')
1539
1540    def detect_sqlite(self):
1541        # The sqlite interface
1542        sqlite_setup_debug = False   # verbose debug prints from this script?
1543
1544        # We hunt for #define SQLITE_VERSION "n.n.n"
1545        sqlite_incdir = sqlite_libdir = None
1546        sqlite_inc_paths = [ '/usr/include',
1547                             '/usr/include/sqlite',
1548                             '/usr/include/sqlite3',
1549                             '/usr/local/include',
1550                             '/usr/local/include/sqlite',
1551                             '/usr/local/include/sqlite3',
1552                             ]
1553        if CROSS_COMPILING:
1554            sqlite_inc_paths = []
1555        MIN_SQLITE_VERSION_NUMBER = (3, 7, 15)  # Issue 40810
1556        MIN_SQLITE_VERSION = ".".join([str(x)
1557                                    for x in MIN_SQLITE_VERSION_NUMBER])
1558
1559        # Scan the default include directories before the SQLite specific
1560        # ones. This allows one to override the copy of sqlite on OSX,
1561        # where /usr/include contains an old version of sqlite.
1562        if MACOS:
1563            sysroot = macosx_sdk_root()
1564
1565        for d_ in self.inc_dirs + sqlite_inc_paths:
1566            d = d_
1567            if MACOS and is_macosx_sdk_path(d):
1568                d = os.path.join(sysroot, d[1:])
1569
1570            f = os.path.join(d, "sqlite3.h")
1571            if os.path.exists(f):
1572                if sqlite_setup_debug: print("sqlite: found %s"%f)
1573                with open(f) as file:
1574                    incf = file.read()
1575                m = re.search(
1576                    r'\s*.*#\s*.*define\s.*SQLITE_VERSION\W*"([\d\.]*)"', incf)
1577                if m:
1578                    sqlite_version = m.group(1)
1579                    sqlite_version_tuple = tuple([int(x)
1580                                        for x in sqlite_version.split(".")])
1581                    if sqlite_version_tuple >= MIN_SQLITE_VERSION_NUMBER:
1582                        # we win!
1583                        if sqlite_setup_debug:
1584                            print("%s/sqlite3.h: version %s"%(d, sqlite_version))
1585                        sqlite_incdir = d
1586                        break
1587                    else:
1588                        if sqlite_setup_debug:
1589                            print("%s: version %s is too old, need >= %s"%(d,
1590                                        sqlite_version, MIN_SQLITE_VERSION))
1591                elif sqlite_setup_debug:
1592                    print("sqlite: %s had no SQLITE_VERSION"%(f,))
1593
1594        if sqlite_incdir:
1595            sqlite_dirs_to_check = [
1596                os.path.join(sqlite_incdir, '..', 'lib64'),
1597                os.path.join(sqlite_incdir, '..', 'lib'),
1598                os.path.join(sqlite_incdir, '..', '..', 'lib64'),
1599                os.path.join(sqlite_incdir, '..', '..', 'lib'),
1600            ]
1601            sqlite_libfile = self.compiler.find_library_file(
1602                                sqlite_dirs_to_check + self.lib_dirs, 'sqlite3')
1603            if sqlite_libfile:
1604                sqlite_libdir = [os.path.abspath(os.path.dirname(sqlite_libfile))]
1605
1606        if sqlite_incdir and sqlite_libdir:
1607            sqlite_srcs = ['_sqlite/cache.c',
1608                '_sqlite/connection.c',
1609                '_sqlite/cursor.c',
1610                '_sqlite/microprotocols.c',
1611                '_sqlite/module.c',
1612                '_sqlite/prepare_protocol.c',
1613                '_sqlite/row.c',
1614                '_sqlite/statement.c',
1615                '_sqlite/util.c', ]
1616            sqlite_defines = []
1617
1618            # Enable support for loadable extensions in the sqlite3 module
1619            # if --enable-loadable-sqlite-extensions configure option is used.
1620            if '--enable-loadable-sqlite-extensions' not in sysconfig.get_config_var("CONFIG_ARGS"):
1621                sqlite_defines.append(("SQLITE_OMIT_LOAD_EXTENSION", "1"))
1622            elif MACOS and sqlite_incdir == os.path.join(MACOS_SDK_ROOT, "usr/include"):
1623                raise DistutilsError("System version of SQLite does not support loadable extensions")
1624
1625            if MACOS:
1626                # In every directory on the search path search for a dynamic
1627                # library and then a static library, instead of first looking
1628                # for dynamic libraries on the entire path.
1629                # This way a statically linked custom sqlite gets picked up
1630                # before the dynamic library in /usr/lib.
1631                sqlite_extra_link_args = ('-Wl,-search_paths_first',)
1632            else:
1633                sqlite_extra_link_args = ()
1634
1635            include_dirs = ["Modules/_sqlite"]
1636            # Only include the directory where sqlite was found if it does
1637            # not already exist in set include directories, otherwise you
1638            # can end up with a bad search path order.
1639            if sqlite_incdir not in self.compiler.include_dirs:
1640                include_dirs.append(sqlite_incdir)
1641            # avoid a runtime library path for a system library dir
1642            if sqlite_libdir and sqlite_libdir[0] in self.lib_dirs:
1643                sqlite_libdir = None
1644            self.add(Extension('_sqlite3', sqlite_srcs,
1645                               define_macros=sqlite_defines,
1646                               include_dirs=include_dirs,
1647                               library_dirs=sqlite_libdir,
1648                               extra_link_args=sqlite_extra_link_args,
1649                               libraries=["sqlite3",]))
1650        else:
1651            self.missing.append('_sqlite3')
1652
1653    def detect_platform_specific_exts(self):
1654        # Unix-only modules
1655        if not MS_WINDOWS:
1656            if not VXWORKS:
1657                # Steen Lumholt's termios module
1658                self.add(Extension('termios', ['termios.c']))
1659                # Jeremy Hylton's rlimit interface
1660            self.add(Extension('resource', ['resource.c']))
1661        else:
1662            self.missing.extend(['resource', 'termios'])
1663
1664        # Platform-specific libraries
1665        if HOST_PLATFORM.startswith(('linux', 'freebsd', 'gnukfreebsd')):
1666            self.add(Extension('ossaudiodev', ['ossaudiodev.c']))
1667        elif not AIX:
1668            self.missing.append('ossaudiodev')
1669
1670        if MACOS:
1671            self.add(Extension('_scproxy', ['_scproxy.c'],
1672                               extra_link_args=[
1673                                   '-framework', 'SystemConfiguration',
1674                                   '-framework', 'CoreFoundation']))
1675
1676    def detect_compress_exts(self):
1677        # Andrew Kuchling's zlib module.  Note that some versions of zlib
1678        # 1.1.3 have security problems.  See CERT Advisory CA-2002-07:
1679        # http://www.cert.org/advisories/CA-2002-07.html
1680        #
1681        # zlib 1.1.4 is fixed, but at least one vendor (RedHat) has decided to
1682        # patch its zlib 1.1.3 package instead of upgrading to 1.1.4.  For
1683        # now, we still accept 1.1.3, because we think it's difficult to
1684        # exploit this in Python, and we'd rather make it RedHat's problem
1685        # than our problem <wink>.
1686        #
1687        # You can upgrade zlib to version 1.1.4 yourself by going to
1688        # http://www.gzip.org/zlib/
1689        zlib_inc = find_file('zlib.h', [], self.inc_dirs)
1690        have_zlib = False
1691        if zlib_inc is not None:
1692            zlib_h = zlib_inc[0] + '/zlib.h'
1693            version = '"0.0.0"'
1694            version_req = '"1.1.3"'
1695            if MACOS and is_macosx_sdk_path(zlib_h):
1696                zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:])
1697            with open(zlib_h) as fp:
1698                while 1:
1699                    line = fp.readline()
1700                    if not line:
1701                        break
1702                    if line.startswith('#define ZLIB_VERSION'):
1703                        version = line.split()[2]
1704                        break
1705            if version >= version_req:
1706                if (self.compiler.find_library_file(self.lib_dirs, 'z')):
1707                    if MACOS:
1708                        zlib_extra_link_args = ('-Wl,-search_paths_first',)
1709                    else:
1710                        zlib_extra_link_args = ()
1711                    self.add(Extension('zlib', ['zlibmodule.c'],
1712                                       libraries=['z'],
1713                                       extra_link_args=zlib_extra_link_args))
1714                    have_zlib = True
1715                else:
1716                    self.missing.append('zlib')
1717            else:
1718                self.missing.append('zlib')
1719        else:
1720            self.missing.append('zlib')
1721
1722        # Helper module for various ascii-encoders.  Uses zlib for an optimized
1723        # crc32 if we have it.  Otherwise binascii uses its own.
1724        if have_zlib:
1725            extra_compile_args = ['-DUSE_ZLIB_CRC32']
1726            libraries = ['z']
1727            extra_link_args = zlib_extra_link_args
1728        else:
1729            extra_compile_args = []
1730            libraries = []
1731            extra_link_args = []
1732        self.add(Extension('binascii', ['binascii.c'],
1733                           extra_compile_args=extra_compile_args,
1734                           libraries=libraries,
1735                           extra_link_args=extra_link_args))
1736
1737        # Gustavo Niemeyer's bz2 module.
1738        if (self.compiler.find_library_file(self.lib_dirs, 'bz2')):
1739            if MACOS:
1740                bz2_extra_link_args = ('-Wl,-search_paths_first',)
1741            else:
1742                bz2_extra_link_args = ()
1743            self.add(Extension('_bz2', ['_bz2module.c'],
1744                               libraries=['bz2'],
1745                               extra_link_args=bz2_extra_link_args))
1746        else:
1747            self.missing.append('_bz2')
1748
1749        # LZMA compression support.
1750        if self.compiler.find_library_file(self.lib_dirs, 'lzma'):
1751            self.add(Extension('_lzma', ['_lzmamodule.c'],
1752                               libraries=['lzma']))
1753        else:
1754            self.missing.append('_lzma')
1755
1756    def detect_expat_elementtree(self):
1757        # Interface to the Expat XML parser
1758        #
1759        # Expat was written by James Clark and is now maintained by a group of
1760        # developers on SourceForge; see www.libexpat.org for more information.
1761        # The pyexpat module was written by Paul Prescod after a prototype by
1762        # Jack Jansen.  The Expat source is included in Modules/expat/.  Usage
1763        # of a system shared libexpat.so is possible with --with-system-expat
1764        # configure option.
1765        #
1766        # More information on Expat can be found at www.libexpat.org.
1767        #
1768        if '--with-system-expat' in sysconfig.get_config_var("CONFIG_ARGS"):
1769            expat_inc = []
1770            define_macros = []
1771            extra_compile_args = []
1772            expat_lib = ['expat']
1773            expat_sources = []
1774            expat_depends = []
1775        else:
1776            expat_inc = [os.path.join(self.srcdir, 'Modules', 'expat')]
1777            define_macros = [
1778                ('HAVE_EXPAT_CONFIG_H', '1'),
1779                # bpo-30947: Python uses best available entropy sources to
1780                # call XML_SetHashSalt(), expat entropy sources are not needed
1781                ('XML_POOR_ENTROPY', '1'),
1782            ]
1783            extra_compile_args = []
1784            # bpo-44394: libexpat uses isnan() of math.h and needs linkage
1785            # against the libm
1786            expat_lib = ['m']
1787            expat_sources = ['expat/xmlparse.c',
1788                             'expat/xmlrole.c',
1789                             'expat/xmltok.c']
1790            expat_depends = ['expat/ascii.h',
1791                             'expat/asciitab.h',
1792                             'expat/expat.h',
1793                             'expat/expat_config.h',
1794                             'expat/expat_external.h',
1795                             'expat/internal.h',
1796                             'expat/latin1tab.h',
1797                             'expat/utf8tab.h',
1798                             'expat/xmlrole.h',
1799                             'expat/xmltok.h',
1800                             'expat/xmltok_impl.h'
1801                             ]
1802
1803            cc = sysconfig.get_config_var('CC').split()[0]
1804            ret = run_command(
1805                      '"%s" -Werror -Wno-unreachable-code -E -xc /dev/null >/dev/null 2>&1' % cc)
1806            if ret == 0:
1807                extra_compile_args.append('-Wno-unreachable-code')
1808
1809        self.add(Extension('pyexpat',
1810                           define_macros=define_macros,
1811                           extra_compile_args=extra_compile_args,
1812                           include_dirs=expat_inc,
1813                           libraries=expat_lib,
1814                           sources=['pyexpat.c'] + expat_sources,
1815                           depends=expat_depends))
1816
1817        # Fredrik Lundh's cElementTree module.  Note that this also
1818        # uses expat (via the CAPI hook in pyexpat).
1819
1820        if os.path.isfile(os.path.join(self.srcdir, 'Modules', '_elementtree.c')):
1821            define_macros.append(('USE_PYEXPAT_CAPI', None))
1822            self.add(Extension('_elementtree',
1823                               define_macros=define_macros,
1824                               include_dirs=expat_inc,
1825                               libraries=expat_lib,
1826                               sources=['_elementtree.c'],
1827                               depends=['pyexpat.c', *expat_sources,
1828                                        *expat_depends]))
1829        else:
1830            self.missing.append('_elementtree')
1831
1832    def detect_multibytecodecs(self):
1833        # Hye-Shik Chang's CJKCodecs modules.
1834        self.add(Extension('_multibytecodec',
1835                           ['cjkcodecs/multibytecodec.c']))
1836        for loc in ('kr', 'jp', 'cn', 'tw', 'hk', 'iso2022'):
1837            self.add(Extension('_codecs_%s' % loc,
1838                               ['cjkcodecs/_codecs_%s.c' % loc]))
1839
1840    def detect_multiprocessing(self):
1841        # Richard Oudkerk's multiprocessing module
1842        if MS_WINDOWS:
1843            multiprocessing_srcs = ['_multiprocessing/multiprocessing.c',
1844                                    '_multiprocessing/semaphore.c']
1845        else:
1846            multiprocessing_srcs = ['_multiprocessing/multiprocessing.c']
1847            if (sysconfig.get_config_var('HAVE_SEM_OPEN') and not
1848                sysconfig.get_config_var('POSIX_SEMAPHORES_NOT_ENABLED')):
1849                multiprocessing_srcs.append('_multiprocessing/semaphore.c')
1850        self.add(Extension('_multiprocessing', multiprocessing_srcs,
1851                           include_dirs=["Modules/_multiprocessing"]))
1852
1853        if (not MS_WINDOWS and
1854           sysconfig.get_config_var('HAVE_SHM_OPEN') and
1855           sysconfig.get_config_var('HAVE_SHM_UNLINK')):
1856            posixshmem_srcs = ['_multiprocessing/posixshmem.c']
1857            libs = []
1858            if sysconfig.get_config_var('SHM_NEEDS_LIBRT'):
1859                # need to link with librt to get shm_open()
1860                libs.append('rt')
1861            self.add(Extension('_posixshmem', posixshmem_srcs,
1862                               define_macros={},
1863                               libraries=libs,
1864                               include_dirs=["Modules/_multiprocessing"]))
1865        else:
1866            self.missing.append('_posixshmem')
1867
1868    def detect_uuid(self):
1869        # Build the _uuid module if possible
1870        uuid_h = sysconfig.get_config_var("HAVE_UUID_H")
1871        uuid_uuid_h = sysconfig.get_config_var("HAVE_UUID_UUID_H")
1872        if uuid_h or uuid_uuid_h:
1873            if sysconfig.get_config_var("HAVE_LIBUUID"):
1874                uuid_libs = ["uuid"]
1875            else:
1876                uuid_libs = []
1877            self.add(Extension('_uuid', ['_uuidmodule.c'],
1878                               libraries=uuid_libs))
1879        else:
1880            self.missing.append('_uuid')
1881
1882    def detect_modules(self):
1883        self.detect_simple_extensions()
1884        if TEST_EXTENSIONS:
1885            self.detect_test_extensions()
1886        self.detect_readline_curses()
1887        self.detect_crypt()
1888        self.detect_socket()
1889        self.detect_openssl_hashlib()
1890        self.detect_hash_builtins()
1891        self.detect_dbm_gdbm()
1892        self.detect_sqlite()
1893        self.detect_platform_specific_exts()
1894        self.detect_nis()
1895        self.detect_compress_exts()
1896        self.detect_expat_elementtree()
1897        self.detect_multibytecodecs()
1898        self.detect_decimal()
1899        self.detect_ctypes()
1900        self.detect_multiprocessing()
1901        if not self.detect_tkinter():
1902            self.missing.append('_tkinter')
1903        self.detect_uuid()
1904
1905##         # Uncomment these lines if you want to play with xxmodule.c
1906##         self.add(Extension('xx', ['xxmodule.c']))
1907
1908        # The limited C API is not compatible with the Py_TRACE_REFS macro.
1909        if not sysconfig.get_config_var('Py_TRACE_REFS'):
1910            self.add(Extension('xxlimited', ['xxlimited.c']))
1911            self.add(Extension('xxlimited_35', ['xxlimited_35.c']))
1912
1913    def detect_tkinter_fromenv(self):
1914        # Build _tkinter using the Tcl/Tk locations specified by
1915        # the _TCLTK_INCLUDES and _TCLTK_LIBS environment variables.
1916        # This method is meant to be invoked by detect_tkinter().
1917        #
1918        # The variables can be set via one of the following ways.
1919        #
1920        # - Automatically, at configuration time, by using pkg-config.
1921        #   The tool is called by the configure script.
1922        #   Additional pkg-config configuration paths can be set via the
1923        #   PKG_CONFIG_PATH environment variable.
1924        #
1925        #     PKG_CONFIG_PATH=".../lib/pkgconfig" ./configure ...
1926        #
1927        # - Explicitly, at configuration time by setting both
1928        #   --with-tcltk-includes and --with-tcltk-libs.
1929        #
1930        #     ./configure ... \
1931        #     --with-tcltk-includes="-I/path/to/tclincludes \
1932        #                            -I/path/to/tkincludes"
1933        #     --with-tcltk-libs="-L/path/to/tcllibs -ltclm.n \
1934        #                        -L/path/to/tklibs -ltkm.n"
1935        #
1936        #  - Explicitly, at compile time, by passing TCLTK_INCLUDES and
1937        #    TCLTK_LIBS to the make target.
1938        #    This will override any configuration-time option.
1939        #
1940        #      make TCLTK_INCLUDES="..." TCLTK_LIBS="..."
1941        #
1942        # This can be useful for building and testing tkinter with multiple
1943        # versions of Tcl/Tk.  Note that a build of Tk depends on a particular
1944        # build of Tcl so you need to specify both arguments and use care when
1945        # overriding.
1946
1947        # The _TCLTK variables are created in the Makefile sharedmods target.
1948        tcltk_includes = os.environ.get('_TCLTK_INCLUDES')
1949        tcltk_libs = os.environ.get('_TCLTK_LIBS')
1950        if not (tcltk_includes and tcltk_libs):
1951            # Resume default configuration search.
1952            return False
1953
1954        extra_compile_args = tcltk_includes.split()
1955        extra_link_args = tcltk_libs.split()
1956        self.add(Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'],
1957                           define_macros=[('WITH_APPINIT', 1)],
1958                           extra_compile_args = extra_compile_args,
1959                           extra_link_args = extra_link_args))
1960        return True
1961
1962    def detect_tkinter_darwin(self):
1963        # Build default _tkinter on macOS using Tcl and Tk frameworks.
1964        # This method is meant to be invoked by detect_tkinter().
1965        #
1966        # The macOS native Tk (AKA Aqua Tk) and Tcl are most commonly
1967        # built and installed as macOS framework bundles.  However,
1968        # for several reasons, we cannot take full advantage of the
1969        # Apple-supplied compiler chain's -framework options here.
1970        # Instead, we need to find and pass to the compiler the
1971        # absolute paths of the Tcl and Tk headers files we want to use
1972        # and the absolute path to the directory containing the Tcl
1973        # and Tk frameworks for linking.
1974        #
1975        # We want to handle here two common use cases on macOS:
1976        # 1. Build and link with system-wide third-party or user-built
1977        #    Tcl and Tk frameworks installed in /Library/Frameworks.
1978        # 2. Build and link using a user-specified macOS SDK so that the
1979        #    built Python can be exported to other systems.  In this case,
1980        #    search only the SDK's /Library/Frameworks (normally empty)
1981        #    and /System/Library/Frameworks.
1982        #
1983        # Any other use cases are handled either by detect_tkinter_fromenv(),
1984        # or detect_tkinter(). The former handles non-standard locations of
1985        # Tcl/Tk, defined via the _TCLTK_INCLUDES and _TCLTK_LIBS environment
1986        # variables. The latter handles any Tcl/Tk versions installed in
1987        # standard Unix directories.
1988        #
1989        # It would be desirable to also handle here the case where
1990        # you want to build and link with a framework build of Tcl and Tk
1991        # that is not in /Library/Frameworks, say, in your private
1992        # $HOME/Library/Frameworks directory or elsewhere. It turns
1993        # out to be difficult to make that work automatically here
1994        # without bringing into play more tools and magic. That case
1995        # can be handled using a recipe with the right arguments
1996        # to detect_tkinter_fromenv().
1997        #
1998        # Note also that the fallback case here is to try to use the
1999        # Apple-supplied Tcl and Tk frameworks in /System/Library but
2000        # be forewarned that they are deprecated by Apple and typically
2001        # out-of-date and buggy; their use should be avoided if at
2002        # all possible by installing a newer version of Tcl and Tk in
2003        # /Library/Frameworks before building Python without
2004        # an explicit SDK or by configuring build arguments explicitly.
2005
2006        from os.path import join, exists
2007
2008        sysroot = macosx_sdk_root() # path to the SDK or '/'
2009
2010        if macosx_sdk_specified():
2011            # Use case #2: an SDK other than '/' was specified.
2012            # Only search there.
2013            framework_dirs = [
2014                join(sysroot, 'Library', 'Frameworks'),
2015                join(sysroot, 'System', 'Library', 'Frameworks'),
2016            ]
2017        else:
2018            # Use case #1: no explicit SDK selected.
2019            # Search the local system-wide /Library/Frameworks,
2020            # not the one in the default SDK, otherwise fall back to
2021            # /System/Library/Frameworks whose header files may be in
2022            # the default SDK or, on older systems, actually installed.
2023            framework_dirs = [
2024                join('/', 'Library', 'Frameworks'),
2025                join(sysroot, 'System', 'Library', 'Frameworks'),
2026            ]
2027
2028        # Find the directory that contains the Tcl.framework and
2029        # Tk.framework bundles.
2030        for F in framework_dirs:
2031            # both Tcl.framework and Tk.framework should be present
2032            for fw in 'Tcl', 'Tk':
2033                if not exists(join(F, fw + '.framework')):
2034                    break
2035            else:
2036                # ok, F is now directory with both frameworks. Continue
2037                # building
2038                break
2039        else:
2040            # Tk and Tcl frameworks not found. Normal "unix" tkinter search
2041            # will now resume.
2042            return False
2043
2044        include_dirs = [
2045            join(F, fw + '.framework', H)
2046            for fw in ('Tcl', 'Tk')
2047            for H in ('Headers',)
2048        ]
2049
2050        # Add the base framework directory as well
2051        compile_args = ['-F', F]
2052
2053        # Do not build tkinter for archs that this Tk was not built with.
2054        cflags = sysconfig.get_config_vars('CFLAGS')[0]
2055        archs = re.findall(r'-arch\s+(\w+)', cflags)
2056
2057        tmpfile = os.path.join(self.build_temp, 'tk.arch')
2058        if not os.path.exists(self.build_temp):
2059            os.makedirs(self.build_temp)
2060
2061        run_command(
2062            "file {}/Tk.framework/Tk | grep 'for architecture' > {}".format(F, tmpfile)
2063        )
2064        with open(tmpfile) as fp:
2065            detected_archs = []
2066            for ln in fp:
2067                a = ln.split()[-1]
2068                if a in archs:
2069                    detected_archs.append(ln.split()[-1])
2070        os.unlink(tmpfile)
2071
2072        arch_args = []
2073        for a in detected_archs:
2074            arch_args.append('-arch')
2075            arch_args.append(a)
2076
2077        compile_args += arch_args
2078        link_args = [','.join(['-Wl', '-F', F, '-framework', 'Tcl', '-framework', 'Tk']), *arch_args]
2079
2080        # The X11/xlib.h file bundled in the Tk sources can cause function
2081        # prototype warnings from the compiler. Since we cannot easily fix
2082        # that, suppress the warnings here instead.
2083        if '-Wstrict-prototypes' in cflags.split():
2084            compile_args.append('-Wno-strict-prototypes')
2085
2086        self.add(Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'],
2087                           define_macros=[('WITH_APPINIT', 1)],
2088                           include_dirs=include_dirs,
2089                           libraries=[],
2090                           extra_compile_args=compile_args,
2091                           extra_link_args=link_args))
2092        return True
2093
2094    def detect_tkinter(self):
2095        # The _tkinter module.
2096        #
2097        # Detection of Tcl/Tk is attempted in the following order:
2098        #   - Through environment variables.
2099        #   - Platform specific detection of Tcl/Tk (currently only macOS).
2100        #   - Search of various standard Unix header/library paths.
2101        #
2102        # Detection stops at the first successful method.
2103
2104        # Check for Tcl and Tk at the locations indicated by _TCLTK_INCLUDES
2105        # and _TCLTK_LIBS environment variables.
2106        if self.detect_tkinter_fromenv():
2107            return True
2108
2109        # Rather than complicate the code below, detecting and building
2110        # AquaTk is a separate method. Only one Tkinter will be built on
2111        # Darwin - either AquaTk, if it is found, or X11 based Tk.
2112        if (MACOS and self.detect_tkinter_darwin()):
2113            return True
2114
2115        # Assume we haven't found any of the libraries or include files
2116        # The versions with dots are used on Unix, and the versions without
2117        # dots on Windows, for detection by cygwin.
2118        tcllib = tklib = tcl_includes = tk_includes = None
2119        for version in ['8.6', '86', '8.5', '85', '8.4', '84', '8.3', '83',
2120                        '8.2', '82', '8.1', '81', '8.0', '80']:
2121            tklib = self.compiler.find_library_file(self.lib_dirs,
2122                                                        'tk' + version)
2123            tcllib = self.compiler.find_library_file(self.lib_dirs,
2124                                                         'tcl' + version)
2125            if tklib and tcllib:
2126                # Exit the loop when we've found the Tcl/Tk libraries
2127                break
2128
2129        # Now check for the header files
2130        if tklib and tcllib:
2131            # Check for the include files on Debian and {Free,Open}BSD, where
2132            # they're put in /usr/include/{tcl,tk}X.Y
2133            dotversion = version
2134            if '.' not in dotversion and "bsd" in HOST_PLATFORM.lower():
2135                # OpenBSD and FreeBSD use Tcl/Tk library names like libtcl83.a,
2136                # but the include subdirs are named like .../include/tcl8.3.
2137                dotversion = dotversion[:-1] + '.' + dotversion[-1]
2138            tcl_include_sub = []
2139            tk_include_sub = []
2140            for dir in self.inc_dirs:
2141                tcl_include_sub += [dir + os.sep + "tcl" + dotversion]
2142                tk_include_sub += [dir + os.sep + "tk" + dotversion]
2143            tk_include_sub += tcl_include_sub
2144            tcl_includes = find_file('tcl.h', self.inc_dirs, tcl_include_sub)
2145            tk_includes = find_file('tk.h', self.inc_dirs, tk_include_sub)
2146
2147        if (tcllib is None or tklib is None or
2148            tcl_includes is None or tk_includes is None):
2149            self.announce("INFO: Can't locate Tcl/Tk libs and/or headers", 2)
2150            return False
2151
2152        # OK... everything seems to be present for Tcl/Tk.
2153
2154        include_dirs = []
2155        libs = []
2156        defs = []
2157        added_lib_dirs = []
2158        for dir in tcl_includes + tk_includes:
2159            if dir not in include_dirs:
2160                include_dirs.append(dir)
2161
2162        # Check for various platform-specific directories
2163        if HOST_PLATFORM == 'sunos5':
2164            include_dirs.append('/usr/openwin/include')
2165            added_lib_dirs.append('/usr/openwin/lib')
2166        elif os.path.exists('/usr/X11R6/include'):
2167            include_dirs.append('/usr/X11R6/include')
2168            added_lib_dirs.append('/usr/X11R6/lib64')
2169            added_lib_dirs.append('/usr/X11R6/lib')
2170        elif os.path.exists('/usr/X11R5/include'):
2171            include_dirs.append('/usr/X11R5/include')
2172            added_lib_dirs.append('/usr/X11R5/lib')
2173        else:
2174            # Assume default location for X11
2175            include_dirs.append('/usr/X11/include')
2176            added_lib_dirs.append('/usr/X11/lib')
2177
2178        # If Cygwin, then verify that X is installed before proceeding
2179        if CYGWIN:
2180            x11_inc = find_file('X11/Xlib.h', [], include_dirs)
2181            if x11_inc is None:
2182                return False
2183
2184        # Check for BLT extension
2185        if self.compiler.find_library_file(self.lib_dirs + added_lib_dirs,
2186                                               'BLT8.0'):
2187            defs.append( ('WITH_BLT', 1) )
2188            libs.append('BLT8.0')
2189        elif self.compiler.find_library_file(self.lib_dirs + added_lib_dirs,
2190                                                'BLT'):
2191            defs.append( ('WITH_BLT', 1) )
2192            libs.append('BLT')
2193
2194        # Add the Tcl/Tk libraries
2195        libs.append('tk'+ version)
2196        libs.append('tcl'+ version)
2197
2198        # Finally, link with the X11 libraries (not appropriate on cygwin)
2199        if not CYGWIN:
2200            libs.append('X11')
2201
2202        # XXX handle these, but how to detect?
2203        # *** Uncomment and edit for PIL (TkImaging) extension only:
2204        #       -DWITH_PIL -I../Extensions/Imaging/libImaging  tkImaging.c \
2205        # *** Uncomment and edit for TOGL extension only:
2206        #       -DWITH_TOGL togl.c \
2207        # *** Uncomment these for TOGL extension only:
2208        #       -lGL -lGLU -lXext -lXmu \
2209
2210        self.add(Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'],
2211                           define_macros=[('WITH_APPINIT', 1)] + defs,
2212                           include_dirs=include_dirs,
2213                           libraries=libs,
2214                           library_dirs=added_lib_dirs))
2215        return True
2216
2217    def configure_ctypes(self, ext):
2218        return True
2219
2220    def detect_ctypes(self):
2221        # Thomas Heller's _ctypes module
2222
2223        if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS):
2224            self.use_system_libffi = True
2225        else:
2226            self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS")
2227
2228        include_dirs = []
2229        extra_compile_args = ['-DPy_BUILD_CORE_MODULE']
2230        extra_link_args = []
2231        sources = ['_ctypes/_ctypes.c',
2232                   '_ctypes/callbacks.c',
2233                   '_ctypes/callproc.c',
2234                   '_ctypes/stgdict.c',
2235                   '_ctypes/cfield.c']
2236        depends = ['_ctypes/ctypes.h']
2237
2238        if MACOS:
2239            sources.append('_ctypes/malloc_closure.c')
2240            extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1')
2241            extra_compile_args.append('-DMACOSX')
2242            include_dirs.append('_ctypes/darwin')
2243
2244        elif HOST_PLATFORM == 'sunos5':
2245            # XXX This shouldn't be necessary; it appears that some
2246            # of the assembler code is non-PIC (i.e. it has relocations
2247            # when it shouldn't. The proper fix would be to rewrite
2248            # the assembler code to be PIC.
2249            # This only works with GCC; the Sun compiler likely refuses
2250            # this option. If you want to compile ctypes with the Sun
2251            # compiler, please research a proper solution, instead of
2252            # finding some -z option for the Sun compiler.
2253            extra_link_args.append('-mimpure-text')
2254
2255        elif HOST_PLATFORM.startswith('hp-ux'):
2256            extra_link_args.append('-fPIC')
2257
2258        ext = Extension('_ctypes',
2259                        include_dirs=include_dirs,
2260                        extra_compile_args=extra_compile_args,
2261                        extra_link_args=extra_link_args,
2262                        libraries=[],
2263                        sources=sources,
2264                        depends=depends)
2265        self.add(ext)
2266        if TEST_EXTENSIONS:
2267            # function my_sqrt() needs libm for sqrt()
2268            self.add(Extension('_ctypes_test',
2269                               sources=['_ctypes/_ctypes_test.c'],
2270                               libraries=['m']))
2271
2272        ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR")
2273        ffi_lib = None
2274
2275        ffi_inc_dirs = self.inc_dirs.copy()
2276        if MACOS:
2277            ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi")
2278
2279            if not ffi_inc:
2280                if os.path.exists(ffi_in_sdk):
2281                    ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1")
2282                    ffi_inc = ffi_in_sdk
2283                    ffi_lib = 'ffi'
2284                else:
2285                    # OS X 10.5 comes with libffi.dylib; the include files are
2286                    # in /usr/include/ffi
2287                    ffi_inc_dirs.append('/usr/include/ffi')
2288
2289        if not ffi_inc:
2290            found = find_file('ffi.h', [], ffi_inc_dirs)
2291            if found:
2292                ffi_inc = found[0]
2293        if ffi_inc:
2294            ffi_h = ffi_inc + '/ffi.h'
2295            if not os.path.exists(ffi_h):
2296                ffi_inc = None
2297                print('Header file {} does not exist'.format(ffi_h))
2298        if ffi_lib is None and ffi_inc:
2299            for lib_name in ('ffi', 'ffi_pic'):
2300                if (self.compiler.find_library_file(self.lib_dirs, lib_name)):
2301                    ffi_lib = lib_name
2302                    break
2303
2304        if ffi_inc and ffi_lib:
2305            ffi_headers = glob(os.path.join(ffi_inc, '*.h'))
2306            if grep_headers_for('ffi_prep_cif_var', ffi_headers):
2307                ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1")
2308            if grep_headers_for('ffi_prep_closure_loc', ffi_headers):
2309                ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1")
2310            if grep_headers_for('ffi_closure_alloc', ffi_headers):
2311                ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1")
2312
2313            ext.include_dirs.append(ffi_inc)
2314            ext.libraries.append(ffi_lib)
2315            self.use_system_libffi = True
2316
2317        if sysconfig.get_config_var('HAVE_LIBDL'):
2318            # for dlopen, see bpo-32647
2319            ext.libraries.append('dl')
2320
2321    def detect_decimal(self):
2322        # Stefan Krah's _decimal module
2323        extra_compile_args = []
2324        undef_macros = []
2325        if '--with-system-libmpdec' in sysconfig.get_config_var("CONFIG_ARGS"):
2326            include_dirs = []
2327            libraries = ['mpdec']
2328            sources = ['_decimal/_decimal.c']
2329            depends = ['_decimal/docstrings.h']
2330        else:
2331            include_dirs = [os.path.abspath(os.path.join(self.srcdir,
2332                                                         'Modules',
2333                                                         '_decimal',
2334                                                         'libmpdec'))]
2335            libraries = ['m']
2336            sources = [
2337              '_decimal/_decimal.c',
2338              '_decimal/libmpdec/basearith.c',
2339              '_decimal/libmpdec/constants.c',
2340              '_decimal/libmpdec/context.c',
2341              '_decimal/libmpdec/convolute.c',
2342              '_decimal/libmpdec/crt.c',
2343              '_decimal/libmpdec/difradix2.c',
2344              '_decimal/libmpdec/fnt.c',
2345              '_decimal/libmpdec/fourstep.c',
2346              '_decimal/libmpdec/io.c',
2347              '_decimal/libmpdec/mpalloc.c',
2348              '_decimal/libmpdec/mpdecimal.c',
2349              '_decimal/libmpdec/numbertheory.c',
2350              '_decimal/libmpdec/sixstep.c',
2351              '_decimal/libmpdec/transpose.c',
2352              ]
2353            depends = [
2354              '_decimal/docstrings.h',
2355              '_decimal/libmpdec/basearith.h',
2356              '_decimal/libmpdec/bits.h',
2357              '_decimal/libmpdec/constants.h',
2358              '_decimal/libmpdec/convolute.h',
2359              '_decimal/libmpdec/crt.h',
2360              '_decimal/libmpdec/difradix2.h',
2361              '_decimal/libmpdec/fnt.h',
2362              '_decimal/libmpdec/fourstep.h',
2363              '_decimal/libmpdec/io.h',
2364              '_decimal/libmpdec/mpalloc.h',
2365              '_decimal/libmpdec/mpdecimal.h',
2366              '_decimal/libmpdec/numbertheory.h',
2367              '_decimal/libmpdec/sixstep.h',
2368              '_decimal/libmpdec/transpose.h',
2369              '_decimal/libmpdec/typearith.h',
2370              '_decimal/libmpdec/umodarith.h',
2371              ]
2372
2373        config = {
2374          'x64':     [('CONFIG_64','1'), ('ASM','1')],
2375          'uint128': [('CONFIG_64','1'), ('ANSI','1'), ('HAVE_UINT128_T','1')],
2376          'ansi64':  [('CONFIG_64','1'), ('ANSI','1')],
2377          'ppro':    [('CONFIG_32','1'), ('PPRO','1'), ('ASM','1')],
2378          'ansi32':  [('CONFIG_32','1'), ('ANSI','1')],
2379          'ansi-legacy': [('CONFIG_32','1'), ('ANSI','1'),
2380                          ('LEGACY_COMPILER','1')],
2381          'universal':   [('UNIVERSAL','1')]
2382        }
2383
2384        cc = sysconfig.get_config_var('CC')
2385        sizeof_size_t = sysconfig.get_config_var('SIZEOF_SIZE_T')
2386        machine = os.environ.get('PYTHON_DECIMAL_WITH_MACHINE')
2387
2388        if machine:
2389            # Override automatic configuration to facilitate testing.
2390            define_macros = config[machine]
2391        elif MACOS:
2392            # Universal here means: build with the same options Python
2393            # was built with.
2394            define_macros = config['universal']
2395        elif sizeof_size_t == 8:
2396            if sysconfig.get_config_var('HAVE_GCC_ASM_FOR_X64'):
2397                define_macros = config['x64']
2398            elif sysconfig.get_config_var('HAVE_GCC_UINT128_T'):
2399                define_macros = config['uint128']
2400            else:
2401                define_macros = config['ansi64']
2402        elif sizeof_size_t == 4:
2403            ppro = sysconfig.get_config_var('HAVE_GCC_ASM_FOR_X87')
2404            if ppro and ('gcc' in cc or 'clang' in cc) and \
2405               not 'sunos' in HOST_PLATFORM:
2406                # solaris: problems with register allocation.
2407                # icc >= 11.0 works as well.
2408                define_macros = config['ppro']
2409                extra_compile_args.append('-Wno-unknown-pragmas')
2410            else:
2411                define_macros = config['ansi32']
2412        else:
2413            raise DistutilsError("_decimal: unsupported architecture")
2414
2415        # Workarounds for toolchain bugs:
2416        if sysconfig.get_config_var('HAVE_IPA_PURE_CONST_BUG'):
2417            # Some versions of gcc miscompile inline asm:
2418            # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491
2419            # https://gcc.gnu.org/ml/gcc/2010-11/msg00366.html
2420            extra_compile_args.append('-fno-ipa-pure-const')
2421        if sysconfig.get_config_var('HAVE_GLIBC_MEMMOVE_BUG'):
2422            # _FORTIFY_SOURCE wrappers for memmove and bcopy are incorrect:
2423            # https://sourceware.org/ml/libc-alpha/2010-12/msg00009.html
2424            undef_macros.append('_FORTIFY_SOURCE')
2425
2426        # Uncomment for extra functionality:
2427        #define_macros.append(('EXTRA_FUNCTIONALITY', 1))
2428        self.add(Extension('_decimal',
2429                           include_dirs=include_dirs,
2430                           libraries=libraries,
2431                           define_macros=define_macros,
2432                           undef_macros=undef_macros,
2433                           extra_compile_args=extra_compile_args,
2434                           sources=sources,
2435                           depends=depends))
2436
2437    def detect_openssl_hashlib(self):
2438        # Detect SSL support for the socket module (via _ssl)
2439        config_vars = sysconfig.get_config_vars()
2440
2441        def split_var(name, sep):
2442            # poor man's shlex, the re module is not available yet.
2443            value = config_vars.get(name)
2444            if not value:
2445                return ()
2446            # This trick works because ax_check_openssl uses --libs-only-L,
2447            # --libs-only-l, and --cflags-only-I.
2448            value = ' ' + value
2449            sep = ' ' + sep
2450            return [v.strip() for v in value.split(sep) if v.strip()]
2451
2452        openssl_includes = split_var('OPENSSL_INCLUDES', '-I')
2453        openssl_libdirs = split_var('OPENSSL_LDFLAGS', '-L')
2454        openssl_libs = split_var('OPENSSL_LIBS', '-l')
2455        openssl_rpath = config_vars.get('OPENSSL_RPATH')
2456        if not openssl_libs:
2457            # libssl and libcrypto not found
2458            self.missing.extend(['_ssl', '_hashlib'])
2459            return None, None
2460
2461        # Find OpenSSL includes
2462        ssl_incs = find_file(
2463            'openssl/ssl.h', self.inc_dirs, openssl_includes
2464        )
2465        if ssl_incs is None:
2466            self.missing.extend(['_ssl', '_hashlib'])
2467            return None, None
2468
2469        if openssl_rpath == 'auto':
2470            runtime_library_dirs = openssl_libdirs[:]
2471        elif not openssl_rpath:
2472            runtime_library_dirs = []
2473        else:
2474            runtime_library_dirs = [openssl_rpath]
2475
2476        openssl_extension_kwargs = dict(
2477            include_dirs=openssl_includes,
2478            library_dirs=openssl_libdirs,
2479            libraries=openssl_libs,
2480            runtime_library_dirs=runtime_library_dirs,
2481        )
2482
2483        # This static linking is NOT OFFICIALLY SUPPORTED.
2484        # Requires static OpenSSL build with position-independent code. Some
2485        # features like DSO engines or external OSSL providers don't work.
2486        # Only tested on GCC and clang on X86_64.
2487        if os.environ.get("PY_UNSUPPORTED_OPENSSL_BUILD") == "static":
2488            extra_linker_args = []
2489            for lib in openssl_extension_kwargs["libraries"]:
2490                # link statically
2491                extra_linker_args.append(f"-l:lib{lib}.a")
2492                # don't export symbols
2493                extra_linker_args.append(f"-Wl,--exclude-libs,lib{lib}.a")
2494            openssl_extension_kwargs["extra_link_args"] = extra_linker_args
2495            # don't link OpenSSL shared libraries.
2496            # include libz for OpenSSL build flavors with compression support
2497            openssl_extension_kwargs["libraries"] = ["z"]
2498
2499        self.add(
2500            Extension(
2501                '_ssl',
2502                ['_ssl.c'],
2503                depends=[
2504                    'socketmodule.h',
2505                    '_ssl.h',
2506                    '_ssl/debughelpers.c',
2507                    '_ssl/misc.c',
2508                    '_ssl/cert.c',
2509                ],
2510                **openssl_extension_kwargs
2511            )
2512        )
2513        self.add(
2514            Extension(
2515                '_hashlib',
2516                ['_hashopenssl.c'],
2517                depends=['hashlib.h'],
2518                **openssl_extension_kwargs,
2519            )
2520        )
2521
2522    def detect_hash_builtins(self):
2523        # By default we always compile these even when OpenSSL is available
2524        # (issue #14693). It's harmless and the object code is tiny
2525        # (40-50 KiB per module, only loaded when actually used).  Modules can
2526        # be disabled via the --with-builtin-hashlib-hashes configure flag.
2527        supported = {"md5", "sha1", "sha256", "sha512", "sha3", "blake2"}
2528
2529        configured = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES")
2530        configured = configured.strip('"').lower()
2531        configured = {
2532            m.strip() for m in configured.split(",")
2533        }
2534
2535        self.disabled_configure.extend(
2536            sorted(supported.difference(configured))
2537        )
2538
2539        if "sha256" in configured:
2540            self.add(Extension(
2541                '_sha256', ['sha256module.c'],
2542                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
2543                depends=['hashlib.h']
2544            ))
2545
2546        if "sha512" in configured:
2547            self.add(Extension(
2548                '_sha512', ['sha512module.c'],
2549                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
2550                depends=['hashlib.h']
2551            ))
2552
2553        if "md5" in configured:
2554            self.add(Extension(
2555                '_md5', ['md5module.c'],
2556                depends=['hashlib.h']
2557            ))
2558
2559        if "sha1" in configured:
2560            self.add(Extension(
2561                '_sha1', ['sha1module.c'],
2562                depends=['hashlib.h']
2563            ))
2564
2565        if "blake2" in configured:
2566            blake2_deps = glob(
2567                os.path.join(escape(self.srcdir), 'Modules/_blake2/impl/*')
2568            )
2569            blake2_deps.append('hashlib.h')
2570            self.add(Extension(
2571                '_blake2',
2572                [
2573                    '_blake2/blake2module.c',
2574                    '_blake2/blake2b_impl.c',
2575                    '_blake2/blake2s_impl.c'
2576                ],
2577                depends=blake2_deps
2578            ))
2579
2580        if "sha3" in configured:
2581            sha3_deps = glob(
2582                os.path.join(escape(self.srcdir), 'Modules/_sha3/kcp/*')
2583            )
2584            sha3_deps.append('hashlib.h')
2585            self.add(Extension(
2586                '_sha3',
2587                ['_sha3/sha3module.c'],
2588                depends=sha3_deps
2589            ))
2590
2591    def detect_nis(self):
2592        if MS_WINDOWS or CYGWIN or HOST_PLATFORM == 'qnx6':
2593            self.missing.append('nis')
2594            return
2595
2596        libs = []
2597        library_dirs = []
2598        includes_dirs = []
2599
2600        # bpo-32521: glibc has deprecated Sun RPC for some time. Fedora 28
2601        # moved headers and libraries to libtirpc and libnsl. The headers
2602        # are in tircp and nsl sub directories.
2603        rpcsvc_inc = find_file(
2604            'rpcsvc/yp_prot.h', self.inc_dirs,
2605            [os.path.join(inc_dir, 'nsl') for inc_dir in self.inc_dirs]
2606        )
2607        rpc_inc = find_file(
2608            'rpc/rpc.h', self.inc_dirs,
2609            [os.path.join(inc_dir, 'tirpc') for inc_dir in self.inc_dirs]
2610        )
2611        if rpcsvc_inc is None or rpc_inc is None:
2612            # not found
2613            self.missing.append('nis')
2614            return
2615        includes_dirs.extend(rpcsvc_inc)
2616        includes_dirs.extend(rpc_inc)
2617
2618        if self.compiler.find_library_file(self.lib_dirs, 'nsl'):
2619            libs.append('nsl')
2620        else:
2621            # libnsl-devel: check for libnsl in nsl/ subdirectory
2622            nsl_dirs = [os.path.join(lib_dir, 'nsl') for lib_dir in self.lib_dirs]
2623            libnsl = self.compiler.find_library_file(nsl_dirs, 'nsl')
2624            if libnsl is not None:
2625                library_dirs.append(os.path.dirname(libnsl))
2626                libs.append('nsl')
2627
2628        if self.compiler.find_library_file(self.lib_dirs, 'tirpc'):
2629            libs.append('tirpc')
2630
2631        self.add(Extension('nis', ['nismodule.c'],
2632                           libraries=libs,
2633                           library_dirs=library_dirs,
2634                           include_dirs=includes_dirs))
2635
2636
2637class PyBuildInstall(install):
2638    # Suppress the warning about installation into the lib_dynload
2639    # directory, which is not in sys.path when running Python during
2640    # installation:
2641    def initialize_options (self):
2642        install.initialize_options(self)
2643        self.warn_dir=0
2644
2645    # Customize subcommands to not install an egg-info file for Python
2646    sub_commands = [('install_lib', install.has_lib),
2647                    ('install_headers', install.has_headers),
2648                    ('install_scripts', install.has_scripts),
2649                    ('install_data', install.has_data)]
2650
2651
2652class PyBuildInstallLib(install_lib):
2653    # Do exactly what install_lib does but make sure correct access modes get
2654    # set on installed directories and files. All installed files with get
2655    # mode 644 unless they are a shared library in which case they will get
2656    # mode 755. All installed directories will get mode 755.
2657
2658    # this is works for EXT_SUFFIX too, which ends with SHLIB_SUFFIX
2659    shlib_suffix = sysconfig.get_config_var("SHLIB_SUFFIX")
2660
2661    def install(self):
2662        outfiles = install_lib.install(self)
2663        self.set_file_modes(outfiles, 0o644, 0o755)
2664        self.set_dir_modes(self.install_dir, 0o755)
2665        return outfiles
2666
2667    def set_file_modes(self, files, defaultMode, sharedLibMode):
2668        if not files: return
2669
2670        for filename in files:
2671            if os.path.islink(filename): continue
2672            mode = defaultMode
2673            if filename.endswith(self.shlib_suffix): mode = sharedLibMode
2674            log.info("changing mode of %s to %o", filename, mode)
2675            if not self.dry_run: os.chmod(filename, mode)
2676
2677    def set_dir_modes(self, dirname, mode):
2678        for dirpath, dirnames, fnames in os.walk(dirname):
2679            if os.path.islink(dirpath):
2680                continue
2681            log.info("changing mode of %s to %o", dirpath, mode)
2682            if not self.dry_run: os.chmod(dirpath, mode)
2683
2684
2685class PyBuildScripts(build_scripts):
2686    def copy_scripts(self):
2687        outfiles, updated_files = build_scripts.copy_scripts(self)
2688        fullversion = '-{0[0]}.{0[1]}'.format(sys.version_info)
2689        minoronly = '.{0[1]}'.format(sys.version_info)
2690        newoutfiles = []
2691        newupdated_files = []
2692        for filename in outfiles:
2693            if filename.endswith('2to3'):
2694                newfilename = filename + fullversion
2695            else:
2696                newfilename = filename + minoronly
2697            log.info(f'renaming {filename} to {newfilename}')
2698            os.rename(filename, newfilename)
2699            newoutfiles.append(newfilename)
2700            if filename in updated_files:
2701                newupdated_files.append(newfilename)
2702        return newoutfiles, newupdated_files
2703
2704
2705def main():
2706    global LIST_MODULE_NAMES
2707
2708    if "--list-module-names" in sys.argv:
2709        LIST_MODULE_NAMES = True
2710        sys.argv.remove("--list-module-names")
2711
2712    set_compiler_flags('CFLAGS', 'PY_CFLAGS_NODIST')
2713    set_compiler_flags('LDFLAGS', 'PY_LDFLAGS_NODIST')
2714
2715    class DummyProcess:
2716        """Hack for parallel build"""
2717        ProcessPoolExecutor = None
2718
2719    sys.modules['concurrent.futures.process'] = DummyProcess
2720    validate_tzpath()
2721
2722    # turn off warnings when deprecated modules are imported
2723    import warnings
2724    warnings.filterwarnings("ignore",category=DeprecationWarning)
2725    setup(# PyPI Metadata (PEP 301)
2726          name = "Python",
2727          version = sys.version.split()[0],
2728          url = "https://www.python.org/%d.%d" % sys.version_info[:2],
2729          maintainer = "Guido van Rossum and the Python community",
2730          maintainer_email = "python-dev@python.org",
2731          description = "A high-level object-oriented programming language",
2732          long_description = SUMMARY.strip(),
2733          license = "PSF license",
2734          classifiers = [x for x in CLASSIFIERS.split("\n") if x],
2735          platforms = ["Many"],
2736
2737          # Build info
2738          cmdclass = {'build_ext': PyBuildExt,
2739                      'build_scripts': PyBuildScripts,
2740                      'install': PyBuildInstall,
2741                      'install_lib': PyBuildInstallLib},
2742          # The struct module is defined here, because build_ext won't be
2743          # called unless there's at least one extension module defined.
2744          ext_modules=[Extension('_struct', ['_struct.c'],
2745                                 extra_compile_args=['-DPy_BUILD_CORE_MODULE'])],
2746
2747          # If you change the scripts installed here, you also need to
2748          # check the PyBuildScripts command above, and change the links
2749          # created by the bininstall target in Makefile.pre.in
2750          scripts = ["Tools/scripts/pydoc3", "Tools/scripts/idle3",
2751                     "Tools/scripts/2to3"]
2752        )
2753
2754# --install-platlib
2755if __name__ == '__main__':
2756    main()
2757