• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""
2Collect various information about Python to help debugging test failures.
3"""
4from __future__ import print_function
5import errno
6import re
7import sys
8import traceback
9import warnings
10
11
12def normalize_text(text):
13    if text is None:
14        return None
15    text = str(text)
16    text = re.sub(r'\s+', ' ', text)
17    return text.strip()
18
19
20class PythonInfo:
21    def __init__(self):
22        self.info = {}
23
24    def add(self, key, value):
25        if key in self.info:
26            raise ValueError("duplicate key: %r" % key)
27
28        if value is None:
29            return
30
31        if not isinstance(value, int):
32            if not isinstance(value, str):
33                # convert other objects like sys.flags to string
34                value = str(value)
35
36            value = value.strip()
37            if not value:
38                return
39
40        self.info[key] = value
41
42    def get_infos(self):
43        """
44        Get information as a key:value dictionary where values are strings.
45        """
46        return {key: str(value) for key, value in self.info.items()}
47
48
49def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None):
50    for attr in attributes:
51        value = getattr(obj, attr, None)
52        if value is None:
53            continue
54        name = name_fmt % attr
55        if formatter is not None:
56            value = formatter(attr, value)
57        info_add(name, value)
58
59
60def copy_attr(info_add, name, mod, attr_name):
61    try:
62        value = getattr(mod, attr_name)
63    except AttributeError:
64        return
65    info_add(name, value)
66
67
68def call_func(info_add, name, mod, func_name, *, formatter=None):
69    try:
70        func = getattr(mod, func_name)
71    except AttributeError:
72        return
73    value = func()
74    if formatter is not None:
75        value = formatter(value)
76    info_add(name, value)
77
78
79def collect_sys(info_add):
80    attributes = (
81        '_framework',
82        'abiflags',
83        'api_version',
84        'builtin_module_names',
85        'byteorder',
86        'dont_write_bytecode',
87        'executable',
88        'flags',
89        'float_info',
90        'float_repr_style',
91        'hash_info',
92        'hexversion',
93        'implementation',
94        'int_info',
95        'maxsize',
96        'maxunicode',
97        'path',
98        'platform',
99        'platlibdir',
100        'prefix',
101        'thread_info',
102        'version',
103        'version_info',
104        'winver',
105    )
106    copy_attributes(info_add, sys, 'sys.%s', attributes)
107
108    call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel')
109    call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion')
110
111    encoding = sys.getfilesystemencoding()
112    if hasattr(sys, 'getfilesystemencodeerrors'):
113        encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors())
114    info_add('sys.filesystem_encoding', encoding)
115
116    for name in ('stdin', 'stdout', 'stderr'):
117        stream = getattr(sys, name)
118        if stream is None:
119            continue
120        encoding = getattr(stream, 'encoding', None)
121        if not encoding:
122            continue
123        errors = getattr(stream, 'errors', None)
124        if errors:
125            encoding = '%s/%s' % (encoding, errors)
126        info_add('sys.%s.encoding' % name, encoding)
127
128    # Were we compiled --with-pydebug or with #define Py_DEBUG?
129    Py_DEBUG = hasattr(sys, 'gettotalrefcount')
130    if Py_DEBUG:
131        text = 'Yes (sys.gettotalrefcount() present)'
132    else:
133        text = 'No (sys.gettotalrefcount() missing)'
134    info_add('Py_DEBUG', text)
135
136
137def collect_platform(info_add):
138    import platform
139
140    arch = platform.architecture()
141    arch = ' '.join(filter(bool, arch))
142    info_add('platform.architecture', arch)
143
144    info_add('platform.python_implementation',
145             platform.python_implementation())
146    info_add('platform.platform',
147             platform.platform(aliased=True))
148
149    libc_ver = ('%s %s' % platform.libc_ver()).strip()
150    if libc_ver:
151        info_add('platform.libc_ver', libc_ver)
152
153
154def collect_locale(info_add):
155    import locale
156
157    info_add('locale.encoding', locale.getpreferredencoding(False))
158
159
160def collect_builtins(info_add):
161    info_add('builtins.float.float_format', float.__getformat__("float"))
162    info_add('builtins.float.double_format', float.__getformat__("double"))
163
164
165def collect_urandom(info_add):
166    import os
167
168    if hasattr(os, 'getrandom'):
169        # PEP 524: Check if system urandom is initialized
170        try:
171            try:
172                os.getrandom(1, os.GRND_NONBLOCK)
173                state = 'ready (initialized)'
174            except BlockingIOError as exc:
175                state = 'not seeded yet (%s)' % exc
176            info_add('os.getrandom', state)
177        except OSError as exc:
178            # Python was compiled on a more recent Linux version
179            # than the current Linux kernel: ignore OSError(ENOSYS)
180            if exc.errno != errno.ENOSYS:
181                raise
182
183
184def collect_os(info_add):
185    import os
186
187    def format_attr(attr, value):
188        if attr in ('supports_follow_symlinks', 'supports_fd',
189                    'supports_effective_ids'):
190            return str(sorted(func.__name__ for func in value))
191        else:
192            return value
193
194    attributes = (
195        'name',
196        'supports_bytes_environ',
197        'supports_effective_ids',
198        'supports_fd',
199        'supports_follow_symlinks',
200    )
201    copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
202
203    for func in (
204        'cpu_count',
205        'getcwd',
206        'getegid',
207        'geteuid',
208        'getgid',
209        'getloadavg',
210        'getresgid',
211        'getresuid',
212        'getuid',
213        'uname',
214    ):
215        call_func(info_add, 'os.%s' % func, os, func)
216
217    def format_groups(groups):
218        return ', '.join(map(str, groups))
219
220    call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups)
221
222    if hasattr(os, 'getlogin'):
223        try:
224            login = os.getlogin()
225        except OSError:
226            # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl
227            # for device" on Travis CI
228            pass
229        else:
230            info_add("os.login", login)
231
232    # Environment variables used by the stdlib and tests. Don't log the full
233    # environment: filter to list to not leak sensitive information.
234    #
235    # HTTP_PROXY is not logged because it can contain a password.
236    ENV_VARS = frozenset((
237        "APPDATA",
238        "AR",
239        "ARCHFLAGS",
240        "ARFLAGS",
241        "AUDIODEV",
242        "CC",
243        "CFLAGS",
244        "COLUMNS",
245        "COMPUTERNAME",
246        "COMSPEC",
247        "CPP",
248        "CPPFLAGS",
249        "DISPLAY",
250        "DISTUTILS_DEBUG",
251        "DISTUTILS_USE_SDK",
252        "DYLD_LIBRARY_PATH",
253        "ENSUREPIP_OPTIONS",
254        "HISTORY_FILE",
255        "HOME",
256        "HOMEDRIVE",
257        "HOMEPATH",
258        "IDLESTARTUP",
259        "LANG",
260        "LDFLAGS",
261        "LDSHARED",
262        "LD_LIBRARY_PATH",
263        "LINES",
264        "MACOSX_DEPLOYMENT_TARGET",
265        "MAILCAPS",
266        "MAKEFLAGS",
267        "MIXERDEV",
268        "MSSDK",
269        "PATH",
270        "PATHEXT",
271        "PIP_CONFIG_FILE",
272        "PLAT",
273        "POSIXLY_CORRECT",
274        "PY_SAX_PARSER",
275        "ProgramFiles",
276        "ProgramFiles(x86)",
277        "RUNNING_ON_VALGRIND",
278        "SDK_TOOLS_BIN",
279        "SERVER_SOFTWARE",
280        "SHELL",
281        "SOURCE_DATE_EPOCH",
282        "SYSTEMROOT",
283        "TEMP",
284        "TERM",
285        "TILE_LIBRARY",
286        "TIX_LIBRARY",
287        "TMP",
288        "TMPDIR",
289        "TRAVIS",
290        "TZ",
291        "USERPROFILE",
292        "VIRTUAL_ENV",
293        "WAYLAND_DISPLAY",
294        "WINDIR",
295        "_PYTHON_HOST_PLATFORM",
296        "_PYTHON_PROJECT_BASE",
297        "_PYTHON_SYSCONFIGDATA_NAME",
298        "__PYVENV_LAUNCHER__",
299    ))
300    for name, value in os.environ.items():
301        uname = name.upper()
302        if (uname in ENV_VARS
303           # Copy PYTHON* and LC_* variables
304           or uname.startswith(("PYTHON", "LC_"))
305           # Visual Studio: VS140COMNTOOLS
306           or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))):
307            info_add('os.environ[%s]' % name, value)
308
309    if hasattr(os, 'umask'):
310        mask = os.umask(0)
311        os.umask(mask)
312        info_add("os.umask", '0o%03o' % mask)
313
314
315def collect_pwd(info_add):
316    try:
317        import pwd
318    except ImportError:
319        return
320    import os
321
322    uid = os.getuid()
323    try:
324        entry = pwd.getpwuid(uid)
325    except KeyError:
326        entry = None
327
328    info_add('pwd.getpwuid(%s)'% uid,
329             entry if entry is not None else '<KeyError>')
330
331    if entry is None:
332        # there is nothing interesting to read if the current user identifier
333        # is not the password database
334        return
335
336    if hasattr(os, 'getgrouplist'):
337        groups = os.getgrouplist(entry.pw_name, entry.pw_gid)
338        groups = ', '.join(map(str, groups))
339        info_add('os.getgrouplist', groups)
340
341
342def collect_readline(info_add):
343    try:
344        import readline
345    except ImportError:
346        return
347
348    def format_attr(attr, value):
349        if isinstance(value, int):
350            return "%#x" % value
351        else:
352            return value
353
354    attributes = (
355        "_READLINE_VERSION",
356        "_READLINE_RUNTIME_VERSION",
357        "_READLINE_LIBRARY_VERSION",
358    )
359    copy_attributes(info_add, readline, 'readline.%s', attributes,
360                    formatter=format_attr)
361
362    if not hasattr(readline, "_READLINE_LIBRARY_VERSION"):
363        # _READLINE_LIBRARY_VERSION has been added to CPython 3.7
364        doc = getattr(readline, '__doc__', '')
365        if 'libedit readline' in doc:
366            info_add('readline.library', 'libedit readline')
367        elif 'GNU readline' in doc:
368            info_add('readline.library', 'GNU readline')
369
370
371def collect_gdb(info_add):
372    import subprocess
373
374    try:
375        proc = subprocess.Popen(["gdb", "-nx", "--version"],
376                                stdout=subprocess.PIPE,
377                                stderr=subprocess.PIPE,
378                                universal_newlines=True)
379        version = proc.communicate()[0]
380        if proc.returncode:
381            # ignore gdb failure: test_gdb will log the error
382            return
383    except OSError:
384        return
385
386    # Only keep the first line
387    version = version.splitlines()[0]
388    info_add('gdb_version', version)
389
390
391def collect_tkinter(info_add):
392    try:
393        import _tkinter
394    except ImportError:
395        pass
396    else:
397        attributes = ('TK_VERSION', 'TCL_VERSION')
398        copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes)
399
400    try:
401        import tkinter
402    except ImportError:
403        pass
404    else:
405        tcl = tkinter.Tcl()
406        patchlevel = tcl.call('info', 'patchlevel')
407        info_add('tkinter.info_patchlevel', patchlevel)
408
409
410def collect_time(info_add):
411    import time
412
413    info_add('time.time', time.time())
414
415    attributes = (
416        'altzone',
417        'daylight',
418        'timezone',
419        'tzname',
420    )
421    copy_attributes(info_add, time, 'time.%s', attributes)
422
423    if hasattr(time, 'get_clock_info'):
424        for clock in ('clock', 'monotonic', 'perf_counter',
425                      'process_time', 'thread_time', 'time'):
426            try:
427                # prevent DeprecatingWarning on get_clock_info('clock')
428                with warnings.catch_warnings(record=True):
429                    clock_info = time.get_clock_info(clock)
430            except ValueError:
431                # missing clock like time.thread_time()
432                pass
433            else:
434                info_add('time.get_clock_info(%s)' % clock, clock_info)
435
436
437def collect_datetime(info_add):
438    try:
439        import datetime
440    except ImportError:
441        return
442
443    info_add('datetime.datetime.now', datetime.datetime.now())
444
445
446def collect_sysconfig(info_add):
447    import sysconfig
448
449    for name in (
450        'ABIFLAGS',
451        'ANDROID_API_LEVEL',
452        'CC',
453        'CCSHARED',
454        'CFLAGS',
455        'CFLAGSFORSHARED',
456        'CONFIG_ARGS',
457        'HOST_GNU_TYPE',
458        'MACHDEP',
459        'MULTIARCH',
460        'OPT',
461        'PY_CFLAGS',
462        'PY_CFLAGS_NODIST',
463        'PY_CORE_LDFLAGS',
464        'PY_LDFLAGS',
465        'PY_LDFLAGS_NODIST',
466        'PY_STDMODULE_CFLAGS',
467        'Py_DEBUG',
468        'Py_ENABLE_SHARED',
469        'SHELL',
470        'SOABI',
471        'prefix',
472    ):
473        value = sysconfig.get_config_var(name)
474        if name == 'ANDROID_API_LEVEL' and not value:
475            # skip ANDROID_API_LEVEL=0
476            continue
477        value = normalize_text(value)
478        info_add('sysconfig[%s]' % name, value)
479
480
481def collect_ssl(info_add):
482    import os
483    try:
484        import ssl
485    except ImportError:
486        return
487    try:
488        import _ssl
489    except ImportError:
490        _ssl = None
491
492    def format_attr(attr, value):
493        if attr.startswith('OP_'):
494            return '%#8x' % value
495        else:
496            return value
497
498    attributes = (
499        'OPENSSL_VERSION',
500        'OPENSSL_VERSION_INFO',
501        'HAS_SNI',
502        'OP_ALL',
503        'OP_NO_TLSv1_1',
504    )
505    copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr)
506
507    for name, ctx in (
508        ('SSLContext', ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)),
509        ('default_https_context', ssl._create_default_https_context()),
510        ('stdlib_context', ssl._create_stdlib_context()),
511    ):
512        attributes = (
513            'minimum_version',
514            'maximum_version',
515            'protocol',
516            'options',
517            'verify_mode',
518        )
519        copy_attributes(info_add, ctx, f'ssl.{name}.%s', attributes)
520
521    env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"]
522    if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'):
523        parts = _ssl.get_default_verify_paths()
524        env_names.extend((parts[0], parts[2]))
525
526    for name in env_names:
527        try:
528            value = os.environ[name]
529        except KeyError:
530            continue
531        info_add('ssl.environ[%s]' % name, value)
532
533
534def collect_socket(info_add):
535    import socket
536
537    hostname = socket.gethostname()
538    info_add('socket.hostname', hostname)
539
540
541def collect_sqlite(info_add):
542    try:
543        import sqlite3
544    except ImportError:
545        return
546
547    attributes = ('version', 'sqlite_version')
548    copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes)
549
550
551def collect_zlib(info_add):
552    try:
553        import zlib
554    except ImportError:
555        return
556
557    attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION')
558    copy_attributes(info_add, zlib, 'zlib.%s', attributes)
559
560
561def collect_expat(info_add):
562    try:
563        from xml.parsers import expat
564    except ImportError:
565        return
566
567    attributes = ('EXPAT_VERSION',)
568    copy_attributes(info_add, expat, 'expat.%s', attributes)
569
570
571def collect_decimal(info_add):
572    try:
573        import _decimal
574    except ImportError:
575        return
576
577    attributes = ('__libmpdec_version__',)
578    copy_attributes(info_add, _decimal, '_decimal.%s', attributes)
579
580
581def collect_testcapi(info_add):
582    try:
583        import _testcapi
584    except ImportError:
585        return
586
587    call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname')
588    copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC')
589
590
591def collect_resource(info_add):
592    try:
593        import resource
594    except ImportError:
595        return
596
597    limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')]
598    for name in limits:
599        key = getattr(resource, name)
600        value = resource.getrlimit(key)
601        info_add('resource.%s' % name, value)
602
603    call_func(info_add, 'resource.pagesize', resource, 'getpagesize')
604
605
606def collect_test_socket(info_add):
607    try:
608        from test import test_socket
609    except ImportError:
610        return
611
612    # all check attributes like HAVE_SOCKET_CAN
613    attributes = [name for name in dir(test_socket)
614                  if name.startswith('HAVE_')]
615    copy_attributes(info_add, test_socket, 'test_socket.%s', attributes)
616
617
618def collect_test_support(info_add):
619    try:
620        from test import support
621    except ImportError:
622        return
623
624    attributes = ('IPV6_ENABLED',)
625    copy_attributes(info_add, support, 'test_support.%s', attributes)
626
627    call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available')
628    call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized')
629
630
631def collect_cc(info_add):
632    import subprocess
633    import sysconfig
634
635    CC = sysconfig.get_config_var('CC')
636    if not CC:
637        return
638
639    try:
640        import shlex
641        args = shlex.split(CC)
642    except ImportError:
643        args = CC.split()
644    args.append('--version')
645    try:
646        proc = subprocess.Popen(args,
647                                stdout=subprocess.PIPE,
648                                stderr=subprocess.STDOUT,
649                                universal_newlines=True)
650    except OSError:
651        # Cannot run the compiler, for example when Python has been
652        # cross-compiled and installed on the target platform where the
653        # compiler is missing.
654        return
655
656    stdout = proc.communicate()[0]
657    if proc.returncode:
658        # CC --version failed: ignore error
659        return
660
661    text = stdout.splitlines()[0]
662    text = normalize_text(text)
663    info_add('CC.version', text)
664
665
666def collect_gdbm(info_add):
667    try:
668        from _gdbm import _GDBM_VERSION
669    except ImportError:
670        return
671
672    info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION)))
673
674
675def collect_get_config(info_add):
676    # Get global configuration variables, _PyPreConfig and _PyCoreConfig
677    try:
678        from _testinternalcapi import get_configs
679    except ImportError:
680        return
681
682    all_configs = get_configs()
683    for config_type in sorted(all_configs):
684        config = all_configs[config_type]
685        for key in sorted(config):
686            info_add('%s[%s]' % (config_type, key), repr(config[key]))
687
688
689def collect_subprocess(info_add):
690    import subprocess
691    copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
692
693
694def collect_windows(info_add):
695    try:
696        import ctypes
697    except ImportError:
698        return
699
700    if not hasattr(ctypes, 'WinDLL'):
701        return
702
703    ntdll = ctypes.WinDLL('ntdll')
704    BOOLEAN = ctypes.c_ubyte
705
706    try:
707        RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
708    except AttributeError:
709        res = '<function not available>'
710    else:
711        RtlAreLongPathsEnabled.restype = BOOLEAN
712        RtlAreLongPathsEnabled.argtypes = ()
713        res = bool(RtlAreLongPathsEnabled())
714    info_add('windows.RtlAreLongPathsEnabled', res)
715
716    try:
717        import _winapi
718        dll_path = _winapi.GetModuleFileName(sys.dllhandle)
719        info_add('windows.dll_path', dll_path)
720    except (ImportError, AttributeError):
721        pass
722
723
724def collect_fips(info_add):
725    try:
726        import _hashlib
727    except ImportError:
728        _hashlib = None
729
730    if _hashlib is not None:
731        call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode')
732
733    try:
734        with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp:
735            line = fp.readline().rstrip()
736
737        if line:
738            info_add('fips.linux_crypto_fips_enabled', line)
739    except OSError:
740        pass
741
742
743def collect_info(info):
744    error = False
745    info_add = info.add
746
747    for collect_func in (
748        # collect_urandom() must be the first, to check the getrandom() status.
749        # Other functions may block on os.urandom() indirectly and so change
750        # its state.
751        collect_urandom,
752
753        collect_builtins,
754        collect_cc,
755        collect_datetime,
756        collect_decimal,
757        collect_expat,
758        collect_fips,
759        collect_gdb,
760        collect_gdbm,
761        collect_get_config,
762        collect_locale,
763        collect_os,
764        collect_platform,
765        collect_pwd,
766        collect_readline,
767        collect_resource,
768        collect_socket,
769        collect_sqlite,
770        collect_ssl,
771        collect_subprocess,
772        collect_sys,
773        collect_sysconfig,
774        collect_testcapi,
775        collect_time,
776        collect_tkinter,
777        collect_windows,
778        collect_zlib,
779
780        # Collecting from tests should be last as they have side effects.
781        collect_test_socket,
782        collect_test_support,
783    ):
784        try:
785            collect_func(info_add)
786        except Exception:
787            error = True
788            print("ERROR: %s() failed" % (collect_func.__name__),
789                  file=sys.stderr)
790            traceback.print_exc(file=sys.stderr)
791            print(file=sys.stderr)
792            sys.stderr.flush()
793
794    return error
795
796
797def dump_info(info, file=None):
798    title = "Python debug information"
799    print(title)
800    print("=" * len(title))
801    print()
802
803    infos = info.get_infos()
804    infos = sorted(infos.items())
805    for key, value in infos:
806        value = value.replace("\n", " ")
807        print("%s: %s" % (key, value))
808    print()
809
810
811def main():
812    info = PythonInfo()
813    error = collect_info(info)
814    dump_info(info)
815
816    if error:
817        print("Collection failed: exit with error", file=sys.stderr)
818        sys.exit(1)
819
820
821if __name__ == "__main__":
822    main()
823