• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""
2Collect various information about Python to help debugging test failures.
3"""
4from __future__ import print_function
5import errno
6import re
7import sys
8import traceback
9import warnings
10
11
12def normalize_text(text):
13    if text is None:
14        return None
15    text = str(text)
16    text = re.sub(r'\s+', ' ', text)
17    return text.strip()
18
19
20class PythonInfo:
21    def __init__(self):
22        self.info = {}
23
24    def add(self, key, value):
25        if key in self.info:
26            raise ValueError("duplicate key: %r" % key)
27
28        if value is None:
29            return
30
31        if not isinstance(value, int):
32            if not isinstance(value, str):
33                # convert other objects like sys.flags to string
34                value = str(value)
35
36            value = value.strip()
37            if not value:
38                return
39
40        self.info[key] = value
41
42    def get_infos(self):
43        """
44        Get information as a key:value dictionary where values are strings.
45        """
46        return {key: str(value) for key, value in self.info.items()}
47
48
49def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None):
50    for attr in attributes:
51        value = getattr(obj, attr, None)
52        if value is None:
53            continue
54        name = name_fmt % attr
55        if formatter is not None:
56            value = formatter(attr, value)
57        info_add(name, value)
58
59
60def copy_attr(info_add, name, mod, attr_name):
61    try:
62        value = getattr(mod, attr_name)
63    except AttributeError:
64        return
65    info_add(name, value)
66
67
68def call_func(info_add, name, mod, func_name, *, formatter=None):
69    try:
70        func = getattr(mod, func_name)
71    except AttributeError:
72        return
73    value = func()
74    if formatter is not None:
75        value = formatter(value)
76    info_add(name, value)
77
78
79def collect_sys(info_add):
80    attributes = (
81        '_framework',
82        'abiflags',
83        'api_version',
84        'builtin_module_names',
85        'byteorder',
86        'dont_write_bytecode',
87        'executable',
88        'flags',
89        'float_info',
90        'float_repr_style',
91        'hash_info',
92        'hexversion',
93        'implementation',
94        'int_info',
95        'maxsize',
96        'maxunicode',
97        'path',
98        'platform',
99        'prefix',
100        'thread_info',
101        'version',
102        'version_info',
103        'winver',
104    )
105    copy_attributes(info_add, sys, 'sys.%s', attributes)
106
107    call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel')
108    call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion')
109
110    encoding = sys.getfilesystemencoding()
111    if hasattr(sys, 'getfilesystemencodeerrors'):
112        encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors())
113    info_add('sys.filesystem_encoding', encoding)
114
115    for name in ('stdin', 'stdout', 'stderr'):
116        stream = getattr(sys, name)
117        if stream is None:
118            continue
119        encoding = getattr(stream, 'encoding', None)
120        if not encoding:
121            continue
122        errors = getattr(stream, 'errors', None)
123        if errors:
124            encoding = '%s/%s' % (encoding, errors)
125        info_add('sys.%s.encoding' % name, encoding)
126
127    # Were we compiled --with-pydebug or with #define Py_DEBUG?
128    Py_DEBUG = hasattr(sys, 'gettotalrefcount')
129    if Py_DEBUG:
130        text = 'Yes (sys.gettotalrefcount() present)'
131    else:
132        text = 'No (sys.gettotalrefcount() missing)'
133    info_add('Py_DEBUG', text)
134
135
136def collect_platform(info_add):
137    import platform
138
139    arch = platform.architecture()
140    arch = ' '.join(filter(bool, arch))
141    info_add('platform.architecture', arch)
142
143    info_add('platform.python_implementation',
144             platform.python_implementation())
145    info_add('platform.platform',
146             platform.platform(aliased=True))
147
148    libc_ver = ('%s %s' % platform.libc_ver()).strip()
149    if libc_ver:
150        info_add('platform.libc_ver', libc_ver)
151
152
153def collect_locale(info_add):
154    import locale
155
156    info_add('locale.encoding', locale.getpreferredencoding(False))
157
158
159def collect_builtins(info_add):
160    info_add('builtins.float.float_format', float.__getformat__("float"))
161    info_add('builtins.float.double_format', float.__getformat__("double"))
162
163
164def collect_urandom(info_add):
165    import os
166
167    if hasattr(os, 'getrandom'):
168        # PEP 524: Check if system urandom is initialized
169        try:
170            try:
171                os.getrandom(1, os.GRND_NONBLOCK)
172                state = 'ready (initialized)'
173            except BlockingIOError as exc:
174                state = 'not seeded yet (%s)' % exc
175            info_add('os.getrandom', state)
176        except OSError as exc:
177            # Python was compiled on a more recent Linux version
178            # than the current Linux kernel: ignore OSError(ENOSYS)
179            if exc.errno != errno.ENOSYS:
180                raise
181
182
183def collect_os(info_add):
184    import os
185
186    def format_attr(attr, value):
187        if attr in ('supports_follow_symlinks', 'supports_fd',
188                    'supports_effective_ids'):
189            return str(sorted(func.__name__ for func in value))
190        else:
191            return value
192
193    attributes = (
194        'name',
195        'supports_bytes_environ',
196        'supports_effective_ids',
197        'supports_fd',
198        'supports_follow_symlinks',
199    )
200    copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
201
202    for func in (
203        'cpu_count',
204        'getcwd',
205        'getegid',
206        'geteuid',
207        'getgid',
208        'getloadavg',
209        'getresgid',
210        'getresuid',
211        'getuid',
212        'uname',
213    ):
214        call_func(info_add, 'os.%s' % func, os, func)
215
216    def format_groups(groups):
217        return ', '.join(map(str, groups))
218
219    call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups)
220
221    if hasattr(os, 'getlogin'):
222        try:
223            login = os.getlogin()
224        except OSError:
225            # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl
226            # for device" on Travis CI
227            pass
228        else:
229            info_add("os.login", login)
230
231    # Environment variables used by the stdlib and tests. Don't log the full
232    # environment: filter to list to not leak sensitive information.
233    #
234    # HTTP_PROXY is not logged because it can contain a password.
235    ENV_VARS = frozenset((
236        "APPDATA",
237        "AR",
238        "ARCHFLAGS",
239        "ARFLAGS",
240        "AUDIODEV",
241        "CC",
242        "CFLAGS",
243        "COLUMNS",
244        "COMPUTERNAME",
245        "COMSPEC",
246        "CPP",
247        "CPPFLAGS",
248        "DISPLAY",
249        "DISTUTILS_DEBUG",
250        "DISTUTILS_USE_SDK",
251        "DYLD_LIBRARY_PATH",
252        "ENSUREPIP_OPTIONS",
253        "HISTORY_FILE",
254        "HOME",
255        "HOMEDRIVE",
256        "HOMEPATH",
257        "IDLESTARTUP",
258        "LANG",
259        "LDFLAGS",
260        "LDSHARED",
261        "LD_LIBRARY_PATH",
262        "LINES",
263        "MACOSX_DEPLOYMENT_TARGET",
264        "MAILCAPS",
265        "MAKEFLAGS",
266        "MIXERDEV",
267        "MSSDK",
268        "PATH",
269        "PATHEXT",
270        "PIP_CONFIG_FILE",
271        "PLAT",
272        "POSIXLY_CORRECT",
273        "PY_SAX_PARSER",
274        "ProgramFiles",
275        "ProgramFiles(x86)",
276        "RUNNING_ON_VALGRIND",
277        "SDK_TOOLS_BIN",
278        "SERVER_SOFTWARE",
279        "SHELL",
280        "SOURCE_DATE_EPOCH",
281        "SYSTEMROOT",
282        "TEMP",
283        "TERM",
284        "TILE_LIBRARY",
285        "TIX_LIBRARY",
286        "TMP",
287        "TMPDIR",
288        "TRAVIS",
289        "TZ",
290        "USERPROFILE",
291        "VIRTUAL_ENV",
292        "WAYLAND_DISPLAY",
293        "WINDIR",
294        "_PYTHON_HOST_PLATFORM",
295        "_PYTHON_PROJECT_BASE",
296        "_PYTHON_SYSCONFIGDATA_NAME",
297        "__PYVENV_LAUNCHER__",
298    ))
299    for name, value in os.environ.items():
300        uname = name.upper()
301        if (uname in ENV_VARS
302           # Copy PYTHON* and LC_* variables
303           or uname.startswith(("PYTHON", "LC_"))
304           # Visual Studio: VS140COMNTOOLS
305           or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))):
306            info_add('os.environ[%s]' % name, value)
307
308    if hasattr(os, 'umask'):
309        mask = os.umask(0)
310        os.umask(mask)
311        info_add("os.umask", '0o%03o' % mask)
312
313
314def collect_pwd(info_add):
315    try:
316        import pwd
317    except ImportError:
318        return
319    import os
320
321    uid = os.getuid()
322    try:
323        entry = pwd.getpwuid(uid)
324    except KeyError:
325        entry = None
326
327    info_add('pwd.getpwuid(%s)'% uid,
328             entry if entry is not None else '<KeyError>')
329
330    if entry is None:
331        # there is nothing interesting to read if the current user identifier
332        # is not the password database
333        return
334
335    if hasattr(os, 'getgrouplist'):
336        groups = os.getgrouplist(entry.pw_name, entry.pw_gid)
337        groups = ', '.join(map(str, groups))
338        info_add('os.getgrouplist', groups)
339
340
341def collect_readline(info_add):
342    try:
343        import readline
344    except ImportError:
345        return
346
347    def format_attr(attr, value):
348        if isinstance(value, int):
349            return "%#x" % value
350        else:
351            return value
352
353    attributes = (
354        "_READLINE_VERSION",
355        "_READLINE_RUNTIME_VERSION",
356        "_READLINE_LIBRARY_VERSION",
357    )
358    copy_attributes(info_add, readline, 'readline.%s', attributes,
359                    formatter=format_attr)
360
361    if not hasattr(readline, "_READLINE_LIBRARY_VERSION"):
362        # _READLINE_LIBRARY_VERSION has been added to CPython 3.7
363        doc = getattr(readline, '__doc__', '')
364        if 'libedit readline' in doc:
365            info_add('readline.library', 'libedit readline')
366        elif 'GNU readline' in doc:
367            info_add('readline.library', 'GNU readline')
368
369
370def collect_gdb(info_add):
371    import subprocess
372
373    try:
374        proc = subprocess.Popen(["gdb", "-nx", "--version"],
375                                stdout=subprocess.PIPE,
376                                stderr=subprocess.PIPE,
377                                universal_newlines=True)
378        version = proc.communicate()[0]
379        if proc.returncode:
380            # ignore gdb failure: test_gdb will log the error
381            return
382    except OSError:
383        return
384
385    # Only keep the first line
386    version = version.splitlines()[0]
387    info_add('gdb_version', version)
388
389
390def collect_tkinter(info_add):
391    try:
392        import _tkinter
393    except ImportError:
394        pass
395    else:
396        attributes = ('TK_VERSION', 'TCL_VERSION')
397        copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes)
398
399    try:
400        import tkinter
401    except ImportError:
402        pass
403    else:
404        tcl = tkinter.Tcl()
405        patchlevel = tcl.call('info', 'patchlevel')
406        info_add('tkinter.info_patchlevel', patchlevel)
407
408
409def collect_time(info_add):
410    import time
411
412    info_add('time.time', time.time())
413
414    attributes = (
415        'altzone',
416        'daylight',
417        'timezone',
418        'tzname',
419    )
420    copy_attributes(info_add, time, 'time.%s', attributes)
421
422    if hasattr(time, 'get_clock_info'):
423        for clock in ('clock', 'monotonic', 'perf_counter',
424                      'process_time', 'thread_time', 'time'):
425            try:
426                # prevent DeprecatingWarning on get_clock_info('clock')
427                with warnings.catch_warnings(record=True):
428                    clock_info = time.get_clock_info(clock)
429            except ValueError:
430                # missing clock like time.thread_time()
431                pass
432            else:
433                info_add('time.get_clock_info(%s)' % clock, clock_info)
434
435
436def collect_datetime(info_add):
437    try:
438        import datetime
439    except ImportError:
440        return
441
442    info_add('datetime.datetime.now', datetime.datetime.now())
443
444
445def collect_sysconfig(info_add):
446    import sysconfig
447
448    for name in (
449        'ABIFLAGS',
450        'ANDROID_API_LEVEL',
451        'CC',
452        'CCSHARED',
453        'CFLAGS',
454        'CFLAGSFORSHARED',
455        'CONFIG_ARGS',
456        'HOST_GNU_TYPE',
457        'MACHDEP',
458        'MULTIARCH',
459        'OPT',
460        'PY_CFLAGS',
461        'PY_CFLAGS_NODIST',
462        'PY_CORE_LDFLAGS',
463        'PY_LDFLAGS',
464        'PY_LDFLAGS_NODIST',
465        'PY_STDMODULE_CFLAGS',
466        'Py_DEBUG',
467        'Py_ENABLE_SHARED',
468        'SHELL',
469        'SOABI',
470        'prefix',
471    ):
472        value = sysconfig.get_config_var(name)
473        if name == 'ANDROID_API_LEVEL' and not value:
474            # skip ANDROID_API_LEVEL=0
475            continue
476        value = normalize_text(value)
477        info_add('sysconfig[%s]' % name, value)
478
479
480def collect_ssl(info_add):
481    import os
482    try:
483        import ssl
484    except ImportError:
485        return
486    try:
487        import _ssl
488    except ImportError:
489        _ssl = None
490
491    def format_attr(attr, value):
492        if attr.startswith('OP_'):
493            return '%#8x' % value
494        else:
495            return value
496
497    attributes = (
498        'OPENSSL_VERSION',
499        'OPENSSL_VERSION_INFO',
500        'HAS_SNI',
501        'OP_ALL',
502        'OP_NO_TLSv1_1',
503    )
504    copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr)
505
506    for name, ctx in (
507        ('SSLContext', ssl.SSLContext()),
508        ('default_https_context', ssl._create_default_https_context()),
509        ('stdlib_context', ssl._create_stdlib_context()),
510    ):
511        attributes = (
512            'minimum_version',
513            'maximum_version',
514            'protocol',
515            'options',
516            'verify_mode',
517        )
518        copy_attributes(info_add, ctx, f'ssl.{name}.%s', attributes)
519
520    env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"]
521    if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'):
522        parts = _ssl.get_default_verify_paths()
523        env_names.extend((parts[0], parts[2]))
524
525    for name in env_names:
526        try:
527            value = os.environ[name]
528        except KeyError:
529            continue
530        info_add('ssl.environ[%s]' % name, value)
531
532
533def collect_socket(info_add):
534    import socket
535
536    hostname = socket.gethostname()
537    info_add('socket.hostname', hostname)
538
539
540def collect_sqlite(info_add):
541    try:
542        import sqlite3
543    except ImportError:
544        return
545
546    attributes = ('version', 'sqlite_version')
547    copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes)
548
549
550def collect_zlib(info_add):
551    try:
552        import zlib
553    except ImportError:
554        return
555
556    attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION')
557    copy_attributes(info_add, zlib, 'zlib.%s', attributes)
558
559
560def collect_expat(info_add):
561    try:
562        from xml.parsers import expat
563    except ImportError:
564        return
565
566    attributes = ('EXPAT_VERSION',)
567    copy_attributes(info_add, expat, 'expat.%s', attributes)
568
569
570def collect_decimal(info_add):
571    try:
572        import _decimal
573    except ImportError:
574        return
575
576    attributes = ('__libmpdec_version__',)
577    copy_attributes(info_add, _decimal, '_decimal.%s', attributes)
578
579
580def collect_testcapi(info_add):
581    try:
582        import _testcapi
583    except ImportError:
584        return
585
586    call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname')
587    copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC')
588
589
590def collect_resource(info_add):
591    try:
592        import resource
593    except ImportError:
594        return
595
596    limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')]
597    for name in limits:
598        key = getattr(resource, name)
599        value = resource.getrlimit(key)
600        info_add('resource.%s' % name, value)
601
602    call_func(info_add, 'resource.pagesize', resource, 'getpagesize')
603
604
605def collect_test_socket(info_add):
606    try:
607        from test import test_socket
608    except ImportError:
609        return
610
611    # all check attributes like HAVE_SOCKET_CAN
612    attributes = [name for name in dir(test_socket)
613                  if name.startswith('HAVE_')]
614    copy_attributes(info_add, test_socket, 'test_socket.%s', attributes)
615
616
617def collect_test_support(info_add):
618    try:
619        from test import support
620    except ImportError:
621        return
622
623    attributes = ('IPV6_ENABLED',)
624    copy_attributes(info_add, support, 'test_support.%s', attributes)
625
626    call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available')
627    call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized')
628
629
630def collect_cc(info_add):
631    import subprocess
632    import sysconfig
633
634    CC = sysconfig.get_config_var('CC')
635    if not CC:
636        return
637
638    try:
639        import shlex
640        args = shlex.split(CC)
641    except ImportError:
642        args = CC.split()
643    args.append('--version')
644    try:
645        proc = subprocess.Popen(args,
646                                stdout=subprocess.PIPE,
647                                stderr=subprocess.STDOUT,
648                                universal_newlines=True)
649    except OSError:
650        # Cannot run the compiler, for example when Python has been
651        # cross-compiled and installed on the target platform where the
652        # compiler is missing.
653        return
654
655    stdout = proc.communicate()[0]
656    if proc.returncode:
657        # CC --version failed: ignore error
658        return
659
660    text = stdout.splitlines()[0]
661    text = normalize_text(text)
662    info_add('CC.version', text)
663
664
665def collect_gdbm(info_add):
666    try:
667        from _gdbm import _GDBM_VERSION
668    except ImportError:
669        return
670
671    info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION)))
672
673
674def collect_get_config(info_add):
675    # Get global configuration variables, _PyPreConfig and _PyCoreConfig
676    try:
677        from _testinternalcapi import get_configs
678    except ImportError:
679        return
680
681    all_configs = get_configs()
682    for config_type in sorted(all_configs):
683        config = all_configs[config_type]
684        for key in sorted(config):
685            info_add('%s[%s]' % (config_type, key), repr(config[key]))
686
687
688def collect_subprocess(info_add):
689    import subprocess
690    copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
691
692
693def collect_windows(info_add):
694    try:
695        import ctypes
696    except ImportError:
697        return
698
699    if not hasattr(ctypes, 'WinDLL'):
700        return
701
702    ntdll = ctypes.WinDLL('ntdll')
703    BOOLEAN = ctypes.c_ubyte
704
705    try:
706        RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
707    except AttributeError:
708        res = '<function not available>'
709    else:
710        RtlAreLongPathsEnabled.restype = BOOLEAN
711        RtlAreLongPathsEnabled.argtypes = ()
712        res = bool(RtlAreLongPathsEnabled())
713    info_add('windows.RtlAreLongPathsEnabled', res)
714
715    try:
716        import _winapi
717        dll_path = _winapi.GetModuleFileName(sys.dllhandle)
718        info_add('windows.dll_path', dll_path)
719    except (ImportError, AttributeError):
720        pass
721
722
723def collect_fips(info_add):
724    try:
725        import _hashlib
726    except ImportError:
727        _hashlib = None
728
729    if _hashlib is not None:
730        call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode')
731
732    try:
733        with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp:
734            line = fp.readline().rstrip()
735
736        if line:
737            info_add('fips.linux_crypto_fips_enabled', line)
738    except OSError:
739        pass
740
741
742def collect_info(info):
743    error = False
744    info_add = info.add
745
746    for collect_func in (
747        # collect_urandom() must be the first, to check the getrandom() status.
748        # Other functions may block on os.urandom() indirectly and so change
749        # its state.
750        collect_urandom,
751
752        collect_builtins,
753        collect_cc,
754        collect_datetime,
755        collect_decimal,
756        collect_expat,
757        collect_fips,
758        collect_gdb,
759        collect_gdbm,
760        collect_get_config,
761        collect_locale,
762        collect_os,
763        collect_platform,
764        collect_pwd,
765        collect_readline,
766        collect_resource,
767        collect_socket,
768        collect_sqlite,
769        collect_ssl,
770        collect_subprocess,
771        collect_sys,
772        collect_sysconfig,
773        collect_testcapi,
774        collect_time,
775        collect_tkinter,
776        collect_windows,
777        collect_zlib,
778
779        # Collecting from tests should be last as they have side effects.
780        collect_test_socket,
781        collect_test_support,
782    ):
783        try:
784            collect_func(info_add)
785        except Exception:
786            error = True
787            print("ERROR: %s() failed" % (collect_func.__name__),
788                  file=sys.stderr)
789            traceback.print_exc(file=sys.stderr)
790            print(file=sys.stderr)
791            sys.stderr.flush()
792
793    return error
794
795
796def dump_info(info, file=None):
797    title = "Python debug information"
798    print(title)
799    print("=" * len(title))
800    print()
801
802    infos = info.get_infos()
803    infos = sorted(infos.items())
804    for key, value in infos:
805        value = value.replace("\n", " ")
806        print("%s: %s" % (key, value))
807    print()
808
809
810def main():
811    info = PythonInfo()
812    error = collect_info(info)
813    dump_info(info)
814
815    if error:
816        print("Collection failed: exit with error", file=sys.stderr)
817        sys.exit(1)
818
819
820if __name__ == "__main__":
821    main()
822