• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""
2Collect various information about Python to help debugging test failures.
3"""
4from __future__ import print_function
5import errno
6import re
7import sys
8import traceback
9import warnings
10
11
12def normalize_text(text):
13    if text is None:
14        return None
15    text = str(text)
16    text = re.sub(r'\s+', ' ', text)
17    return text.strip()
18
19
20class PythonInfo:
21    def __init__(self):
22        self.info = {}
23
24    def add(self, key, value):
25        if key in self.info:
26            raise ValueError("duplicate key: %r" % key)
27
28        if value is None:
29            return
30
31        if not isinstance(value, int):
32            if not isinstance(value, str):
33                # convert other objects like sys.flags to string
34                value = str(value)
35
36            value = value.strip()
37            if not value:
38                return
39
40        self.info[key] = value
41
42    def get_infos(self):
43        """
44        Get information as a key:value dictionary where values are strings.
45        """
46        return {key: str(value) for key, value in self.info.items()}
47
48
49def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None):
50    for attr in attributes:
51        value = getattr(obj, attr, None)
52        if value is None:
53            continue
54        name = name_fmt % attr
55        if formatter is not None:
56            value = formatter(attr, value)
57        info_add(name, value)
58
59
60def copy_attr(info_add, name, mod, attr_name):
61    try:
62        value = getattr(mod, attr_name)
63    except AttributeError:
64        return
65    info_add(name, value)
66
67
68def call_func(info_add, name, mod, func_name, *, formatter=None):
69    try:
70        func = getattr(mod, func_name)
71    except AttributeError:
72        return
73    value = func()
74    if formatter is not None:
75        value = formatter(value)
76    info_add(name, value)
77
78
79def collect_sys(info_add):
80    attributes = (
81        '_framework',
82        'abiflags',
83        'api_version',
84        'builtin_module_names',
85        'byteorder',
86        'dont_write_bytecode',
87        'executable',
88        'flags',
89        'float_info',
90        'float_repr_style',
91        'hash_info',
92        'hexversion',
93        'implementation',
94        'int_info',
95        'maxsize',
96        'maxunicode',
97        'path',
98        'platform',
99        'prefix',
100        'thread_info',
101        'version',
102        'version_info',
103        'winver',
104    )
105    copy_attributes(info_add, sys, 'sys.%s', attributes)
106
107    call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel')
108    call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion')
109
110    encoding = sys.getfilesystemencoding()
111    if hasattr(sys, 'getfilesystemencodeerrors'):
112        encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors())
113    info_add('sys.filesystem_encoding', encoding)
114
115    for name in ('stdin', 'stdout', 'stderr'):
116        stream = getattr(sys, name)
117        if stream is None:
118            continue
119        encoding = getattr(stream, 'encoding', None)
120        if not encoding:
121            continue
122        errors = getattr(stream, 'errors', None)
123        if errors:
124            encoding = '%s/%s' % (encoding, errors)
125        info_add('sys.%s.encoding' % name, encoding)
126
127    # Were we compiled --with-pydebug or with #define Py_DEBUG?
128    Py_DEBUG = hasattr(sys, 'gettotalrefcount')
129    if Py_DEBUG:
130        text = 'Yes (sys.gettotalrefcount() present)'
131    else:
132        text = 'No (sys.gettotalrefcount() missing)'
133    info_add('Py_DEBUG', text)
134
135
136def collect_platform(info_add):
137    import platform
138
139    arch = platform.architecture()
140    arch = ' '.join(filter(bool, arch))
141    info_add('platform.architecture', arch)
142
143    info_add('platform.python_implementation',
144             platform.python_implementation())
145    info_add('platform.platform',
146             platform.platform(aliased=True))
147
148    libc_ver = ('%s %s' % platform.libc_ver()).strip()
149    if libc_ver:
150        info_add('platform.libc_ver', libc_ver)
151
152
153def collect_locale(info_add):
154    import locale
155
156    info_add('locale.encoding', locale.getpreferredencoding(False))
157
158
159def collect_builtins(info_add):
160    info_add('builtins.float.float_format', float.__getformat__("float"))
161    info_add('builtins.float.double_format', float.__getformat__("double"))
162
163
164def collect_urandom(info_add):
165    import os
166
167    if hasattr(os, 'getrandom'):
168        # PEP 524: Check if system urandom is initialized
169        try:
170            try:
171                os.getrandom(1, os.GRND_NONBLOCK)
172                state = 'ready (initialized)'
173            except BlockingIOError as exc:
174                state = 'not seeded yet (%s)' % exc
175            info_add('os.getrandom', state)
176        except OSError as exc:
177            # Python was compiled on a more recent Linux version
178            # than the current Linux kernel: ignore OSError(ENOSYS)
179            if exc.errno != errno.ENOSYS:
180                raise
181
182
183def collect_os(info_add):
184    import os
185
186    def format_attr(attr, value):
187        if attr in ('supports_follow_symlinks', 'supports_fd',
188                    'supports_effective_ids'):
189            return str(sorted(func.__name__ for func in value))
190        else:
191            return value
192
193    attributes = (
194        'name',
195        'supports_bytes_environ',
196        'supports_effective_ids',
197        'supports_fd',
198        'supports_follow_symlinks',
199    )
200    copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
201
202    call_func(info_add, 'os.getcwd', os, 'getcwd')
203
204    call_func(info_add, 'os.getuid', os, 'getuid')
205    call_func(info_add, 'os.getgid', os, 'getgid')
206    call_func(info_add, 'os.uname', os, 'uname')
207
208    def format_groups(groups):
209        return ', '.join(map(str, groups))
210
211    call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups)
212
213    if hasattr(os, 'getlogin'):
214        try:
215            login = os.getlogin()
216        except OSError:
217            # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl
218            # for device" on Travis CI
219            pass
220        else:
221            info_add("os.login", login)
222
223    call_func(info_add, 'os.cpu_count', os, 'cpu_count')
224    call_func(info_add, 'os.getloadavg', os, 'getloadavg')
225
226    # Environment variables used by the stdlib and tests. Don't log the full
227    # environment: filter to list to not leak sensitive information.
228    #
229    # HTTP_PROXY is not logged because it can contain a password.
230    ENV_VARS = frozenset((
231        "APPDATA",
232        "AR",
233        "ARCHFLAGS",
234        "ARFLAGS",
235        "AUDIODEV",
236        "CC",
237        "CFLAGS",
238        "COLUMNS",
239        "COMPUTERNAME",
240        "COMSPEC",
241        "CPP",
242        "CPPFLAGS",
243        "DISPLAY",
244        "DISTUTILS_DEBUG",
245        "DISTUTILS_USE_SDK",
246        "DYLD_LIBRARY_PATH",
247        "ENSUREPIP_OPTIONS",
248        "HISTORY_FILE",
249        "HOME",
250        "HOMEDRIVE",
251        "HOMEPATH",
252        "IDLESTARTUP",
253        "LANG",
254        "LDFLAGS",
255        "LDSHARED",
256        "LD_LIBRARY_PATH",
257        "LINES",
258        "MACOSX_DEPLOYMENT_TARGET",
259        "MAILCAPS",
260        "MAKEFLAGS",
261        "MIXERDEV",
262        "MSSDK",
263        "PATH",
264        "PATHEXT",
265        "PIP_CONFIG_FILE",
266        "PLAT",
267        "POSIXLY_CORRECT",
268        "PY_SAX_PARSER",
269        "ProgramFiles",
270        "ProgramFiles(x86)",
271        "RUNNING_ON_VALGRIND",
272        "SDK_TOOLS_BIN",
273        "SERVER_SOFTWARE",
274        "SHELL",
275        "SOURCE_DATE_EPOCH",
276        "SYSTEMROOT",
277        "TEMP",
278        "TERM",
279        "TILE_LIBRARY",
280        "TIX_LIBRARY",
281        "TMP",
282        "TMPDIR",
283        "TRAVIS",
284        "TZ",
285        "USERPROFILE",
286        "VIRTUAL_ENV",
287        "WAYLAND_DISPLAY",
288        "WINDIR",
289        "_PYTHON_HOST_PLATFORM",
290        "_PYTHON_PROJECT_BASE",
291        "_PYTHON_SYSCONFIGDATA_NAME",
292        "__PYVENV_LAUNCHER__",
293    ))
294    for name, value in os.environ.items():
295        uname = name.upper()
296        if (uname in ENV_VARS
297           # Copy PYTHON* and LC_* variables
298           or uname.startswith(("PYTHON", "LC_"))
299           # Visual Studio: VS140COMNTOOLS
300           or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))):
301            info_add('os.environ[%s]' % name, value)
302
303    if hasattr(os, 'umask'):
304        mask = os.umask(0)
305        os.umask(mask)
306        info_add("os.umask", '%03o' % mask)
307
308
309def collect_pwd(info_add):
310    try:
311        import pwd
312    except ImportError:
313        return
314    import os
315
316    uid = os.getuid()
317    try:
318        entry = pwd.getpwuid(uid)
319    except KeyError:
320        entry = None
321
322    info_add('pwd.getpwuid(%s)'% uid,
323             entry if entry is not None else '<KeyError>')
324
325    if entry is None:
326        # there is nothing interesting to read if the current user identifier
327        # is not the password database
328        return
329
330    if hasattr(os, 'getgrouplist'):
331        groups = os.getgrouplist(entry.pw_name, entry.pw_gid)
332        groups = ', '.join(map(str, groups))
333        info_add('os.getgrouplist', groups)
334
335
336def collect_readline(info_add):
337    try:
338        import readline
339    except ImportError:
340        return
341
342    def format_attr(attr, value):
343        if isinstance(value, int):
344            return "%#x" % value
345        else:
346            return value
347
348    attributes = (
349        "_READLINE_VERSION",
350        "_READLINE_RUNTIME_VERSION",
351        "_READLINE_LIBRARY_VERSION",
352    )
353    copy_attributes(info_add, readline, 'readline.%s', attributes,
354                    formatter=format_attr)
355
356    if not hasattr(readline, "_READLINE_LIBRARY_VERSION"):
357        # _READLINE_LIBRARY_VERSION has been added to CPython 3.7
358        doc = getattr(readline, '__doc__', '')
359        if 'libedit readline' in doc:
360            info_add('readline.library', 'libedit readline')
361        elif 'GNU readline' in doc:
362            info_add('readline.library', 'GNU readline')
363
364
365def collect_gdb(info_add):
366    import subprocess
367
368    try:
369        proc = subprocess.Popen(["gdb", "-nx", "--version"],
370                                stdout=subprocess.PIPE,
371                                stderr=subprocess.PIPE,
372                                universal_newlines=True)
373        version = proc.communicate()[0]
374        if proc.returncode:
375            # ignore gdb failure: test_gdb will log the error
376            return
377    except OSError:
378        return
379
380    # Only keep the first line
381    version = version.splitlines()[0]
382    info_add('gdb_version', version)
383
384
385def collect_tkinter(info_add):
386    try:
387        import _tkinter
388    except ImportError:
389        pass
390    else:
391        attributes = ('TK_VERSION', 'TCL_VERSION')
392        copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes)
393
394    try:
395        import tkinter
396    except ImportError:
397        pass
398    else:
399        tcl = tkinter.Tcl()
400        patchlevel = tcl.call('info', 'patchlevel')
401        info_add('tkinter.info_patchlevel', patchlevel)
402
403
404def collect_time(info_add):
405    import time
406
407    info_add('time.time', time.time())
408
409    attributes = (
410        'altzone',
411        'daylight',
412        'timezone',
413        'tzname',
414    )
415    copy_attributes(info_add, time, 'time.%s', attributes)
416
417    if hasattr(time, 'get_clock_info'):
418        for clock in ('clock', 'monotonic', 'perf_counter',
419                      'process_time', 'thread_time', 'time'):
420            try:
421                # prevent DeprecatingWarning on get_clock_info('clock')
422                with warnings.catch_warnings(record=True):
423                    clock_info = time.get_clock_info(clock)
424            except ValueError:
425                # missing clock like time.thread_time()
426                pass
427            else:
428                info_add('time.get_clock_info(%s)' % clock, clock_info)
429
430
431def collect_datetime(info_add):
432    try:
433        import datetime
434    except ImportError:
435        return
436
437    info_add('datetime.datetime.now', datetime.datetime.now())
438
439
440def collect_sysconfig(info_add):
441    import sysconfig
442
443    for name in (
444        'ABIFLAGS',
445        'ANDROID_API_LEVEL',
446        'CC',
447        'CCSHARED',
448        'CFLAGS',
449        'CFLAGSFORSHARED',
450        'CONFIG_ARGS',
451        'HOST_GNU_TYPE',
452        'MACHDEP',
453        'MULTIARCH',
454        'OPT',
455        'PY_CFLAGS',
456        'PY_CFLAGS_NODIST',
457        'PY_CORE_LDFLAGS',
458        'PY_LDFLAGS',
459        'PY_LDFLAGS_NODIST',
460        'PY_STDMODULE_CFLAGS',
461        'Py_DEBUG',
462        'Py_ENABLE_SHARED',
463        'SHELL',
464        'SOABI',
465        'prefix',
466    ):
467        value = sysconfig.get_config_var(name)
468        if name == 'ANDROID_API_LEVEL' and not value:
469            # skip ANDROID_API_LEVEL=0
470            continue
471        value = normalize_text(value)
472        info_add('sysconfig[%s]' % name, value)
473
474
475def collect_ssl(info_add):
476    import os
477    try:
478        import ssl
479    except ImportError:
480        return
481    try:
482        import _ssl
483    except ImportError:
484        _ssl = None
485
486    def format_attr(attr, value):
487        if attr.startswith('OP_'):
488            return '%#8x' % value
489        else:
490            return value
491
492    attributes = (
493        'OPENSSL_VERSION',
494        'OPENSSL_VERSION_INFO',
495        'HAS_SNI',
496        'OP_ALL',
497        'OP_NO_TLSv1_1',
498    )
499    copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr)
500
501    for name, ctx in (
502        ('SSLContext', ssl.SSLContext()),
503        ('default_https_context', ssl._create_default_https_context()),
504        ('stdlib_context', ssl._create_stdlib_context()),
505    ):
506        attributes = (
507            'minimum_version',
508            'maximum_version',
509            'protocol',
510            'options',
511            'verify_mode',
512        )
513        copy_attributes(info_add, ctx, f'ssl.{name}.%s', attributes)
514
515    env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"]
516    if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'):
517        parts = _ssl.get_default_verify_paths()
518        env_names.extend((parts[0], parts[2]))
519
520    for name in env_names:
521        try:
522            value = os.environ[name]
523        except KeyError:
524            continue
525        info_add('ssl.environ[%s]' % name, value)
526
527
528def collect_socket(info_add):
529    import socket
530
531    hostname = socket.gethostname()
532    info_add('socket.hostname', hostname)
533
534
535def collect_sqlite(info_add):
536    try:
537        import sqlite3
538    except ImportError:
539        return
540
541    attributes = ('version', 'sqlite_version')
542    copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes)
543
544
545def collect_zlib(info_add):
546    try:
547        import zlib
548    except ImportError:
549        return
550
551    attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION')
552    copy_attributes(info_add, zlib, 'zlib.%s', attributes)
553
554
555def collect_expat(info_add):
556    try:
557        from xml.parsers import expat
558    except ImportError:
559        return
560
561    attributes = ('EXPAT_VERSION',)
562    copy_attributes(info_add, expat, 'expat.%s', attributes)
563
564
565def collect_decimal(info_add):
566    try:
567        import _decimal
568    except ImportError:
569        return
570
571    attributes = ('__libmpdec_version__',)
572    copy_attributes(info_add, _decimal, '_decimal.%s', attributes)
573
574
575def collect_testcapi(info_add):
576    try:
577        import _testcapi
578    except ImportError:
579        return
580
581    call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname')
582    copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC')
583
584
585def collect_resource(info_add):
586    try:
587        import resource
588    except ImportError:
589        return
590
591    limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')]
592    for name in limits:
593        key = getattr(resource, name)
594        value = resource.getrlimit(key)
595        info_add('resource.%s' % name, value)
596
597    call_func(info_add, 'resource.pagesize', resource, 'getpagesize')
598
599
600def collect_test_socket(info_add):
601    try:
602        from test import test_socket
603    except ImportError:
604        return
605
606    # all check attributes like HAVE_SOCKET_CAN
607    attributes = [name for name in dir(test_socket)
608                  if name.startswith('HAVE_')]
609    copy_attributes(info_add, test_socket, 'test_socket.%s', attributes)
610
611
612def collect_test_support(info_add):
613    try:
614        from test import support
615    except ImportError:
616        return
617
618    attributes = ('IPV6_ENABLED',)
619    copy_attributes(info_add, support, 'test_support.%s', attributes)
620
621    call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available')
622    call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized')
623
624
625def collect_cc(info_add):
626    import subprocess
627    import sysconfig
628
629    CC = sysconfig.get_config_var('CC')
630    if not CC:
631        return
632
633    try:
634        import shlex
635        args = shlex.split(CC)
636    except ImportError:
637        args = CC.split()
638    args.append('--version')
639    try:
640        proc = subprocess.Popen(args,
641                                stdout=subprocess.PIPE,
642                                stderr=subprocess.STDOUT,
643                                universal_newlines=True)
644    except OSError:
645        # Cannot run the compiler, for example when Python has been
646        # cross-compiled and installed on the target platform where the
647        # compiler is missing.
648        return
649
650    stdout = proc.communicate()[0]
651    if proc.returncode:
652        # CC --version failed: ignore error
653        return
654
655    text = stdout.splitlines()[0]
656    text = normalize_text(text)
657    info_add('CC.version', text)
658
659
660def collect_gdbm(info_add):
661    try:
662        from _gdbm import _GDBM_VERSION
663    except ImportError:
664        return
665
666    info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION)))
667
668
669def collect_get_config(info_add):
670    # Get global configuration variables, _PyPreConfig and _PyCoreConfig
671    try:
672        from _testinternalcapi import get_configs
673    except ImportError:
674        return
675
676    all_configs = get_configs()
677    for config_type in sorted(all_configs):
678        config = all_configs[config_type]
679        for key in sorted(config):
680            info_add('%s[%s]' % (config_type, key), repr(config[key]))
681
682
683def collect_subprocess(info_add):
684    import subprocess
685    copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
686
687
688def collect_windows(info_add):
689    try:
690        import ctypes
691    except ImportError:
692        return
693
694    if not hasattr(ctypes, 'WinDLL'):
695        return
696
697    ntdll = ctypes.WinDLL('ntdll')
698    BOOLEAN = ctypes.c_ubyte
699
700    try:
701        RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
702    except AttributeError:
703        res = '<function not available>'
704    else:
705        RtlAreLongPathsEnabled.restype = BOOLEAN
706        RtlAreLongPathsEnabled.argtypes = ()
707        res = bool(RtlAreLongPathsEnabled())
708    info_add('windows.RtlAreLongPathsEnabled', res)
709
710    try:
711        import _winapi
712        dll_path = _winapi.GetModuleFileName(sys.dllhandle)
713        info_add('windows.dll_path', dll_path)
714    except (ImportError, AttributeError):
715        pass
716
717
718def collect_info(info):
719    error = False
720    info_add = info.add
721
722    for collect_func in (
723        # collect_urandom() must be the first, to check the getrandom() status.
724        # Other functions may block on os.urandom() indirectly and so change
725        # its state.
726        collect_urandom,
727
728        collect_builtins,
729        collect_cc,
730        collect_datetime,
731        collect_decimal,
732        collect_expat,
733        collect_gdb,
734        collect_gdbm,
735        collect_get_config,
736        collect_locale,
737        collect_os,
738        collect_platform,
739        collect_pwd,
740        collect_readline,
741        collect_resource,
742        collect_socket,
743        collect_sqlite,
744        collect_ssl,
745        collect_subprocess,
746        collect_sys,
747        collect_sysconfig,
748        collect_testcapi,
749        collect_time,
750        collect_tkinter,
751        collect_windows,
752        collect_zlib,
753
754        # Collecting from tests should be last as they have side effects.
755        collect_test_socket,
756        collect_test_support,
757    ):
758        try:
759            collect_func(info_add)
760        except Exception as exc:
761            error = True
762            print("ERROR: %s() failed" % (collect_func.__name__),
763                  file=sys.stderr)
764            traceback.print_exc(file=sys.stderr)
765            print(file=sys.stderr)
766            sys.stderr.flush()
767
768    return error
769
770
771def dump_info(info, file=None):
772    title = "Python debug information"
773    print(title)
774    print("=" * len(title))
775    print()
776
777    infos = info.get_infos()
778    infos = sorted(infos.items())
779    for key, value in infos:
780        value = value.replace("\n", " ")
781        print("%s: %s" % (key, value))
782    print()
783
784
785def main():
786    info = PythonInfo()
787    error = collect_info(info)
788    dump_info(info)
789
790    if error:
791        print("Collection failed: exit with error", file=sys.stderr)
792        sys.exit(1)
793
794
795if __name__ == "__main__":
796    main()
797