• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# This script lists the names of standard library modules
2# to update Python/stdlib_module_names.h
3import _imp
4import os.path
5import sys
6import sysconfig
7
8from check_extension_modules import ModuleChecker
9
10
11SCRIPT_NAME = 'Tools/build/generate_stdlib_module_names.py'
12
13SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
14STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
15
16IGNORE = {
17    '__init__',
18    '__pycache__',
19    'site-packages',
20
21    # Test modules and packages
22    '__hello__',
23    '__phello__',
24    '__hello_alias__',
25    '__phello_alias__',
26    '__hello_only__',
27    '_ctypes_test',
28    '_testbuffer',
29    '_testcapi',
30    '_testclinic',
31    '_testclinic_limited',
32    '_testconsole',
33    '_testimportmultiple',
34    '_testinternalcapi',
35    '_testlimitedcapi',
36    '_testmultiphase',
37    '_testsinglephase',
38    '_testexternalinspection',
39    '_xxtestfuzz',
40    'idlelib.idle_test',
41    'test',
42    'xxlimited',
43    'xxlimited_35',
44    'xxsubtype',
45}
46
47ALLOW_TEST_MODULES = {
48    'doctest',
49    'unittest',
50}
51
52# Built-in modules
53def list_builtin_modules(names):
54    names |= set(sys.builtin_module_names)
55
56
57# Pure Python modules (Lib/*.py)
58def list_python_modules(names):
59    for filename in os.listdir(STDLIB_PATH):
60        if not filename.endswith(".py"):
61            continue
62        name = filename.removesuffix(".py")
63        names.add(name)
64
65
66# Packages in Lib/
67def list_packages(names):
68    for name in os.listdir(STDLIB_PATH):
69        if name in IGNORE:
70            continue
71        package_path = os.path.join(STDLIB_PATH, name)
72        if not os.path.isdir(package_path):
73            continue
74        if any(package_file.endswith(".py")
75               for package_file in os.listdir(package_path)):
76            names.add(name)
77
78
79# Built-in and extension modules built by Modules/Setup*
80# includes Windows and macOS extensions.
81def list_modules_setup_extensions(names):
82    checker = ModuleChecker()
83    names.update(checker.list_module_names(all=True))
84
85
86# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
87# Use the "./Programs/_testembed list_frozen" command.
88def list_frozen(names):
89    submodules = set()
90    for name in _imp._frozen_module_names():
91        # To skip __hello__, __hello_alias__ and etc.
92        if name.startswith('__'):
93            continue
94        if '.' in name:
95            submodules.add(name)
96        else:
97            names.add(name)
98    # Make sure all frozen submodules have a known parent.
99    for name in list(submodules):
100        if name.partition('.')[0] in names:
101            submodules.remove(name)
102    if submodules:
103        raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
104
105
106def list_modules():
107    names = set()
108
109    list_builtin_modules(names)
110    list_modules_setup_extensions(names)
111    list_packages(names)
112    list_python_modules(names)
113    list_frozen(names)
114
115    # Remove ignored packages and modules
116    for name in list(names):
117        package_name = name.split('.')[0]
118        # package_name can be equal to name
119        if package_name in IGNORE:
120            names.discard(name)
121
122    # Sanity checks
123    for name in names:
124        if "." in name:
125            raise Exception(f"sub-modules must not be listed: {name}")
126        if ("test" in name or "xx" in name) and name not in ALLOW_TEST_MODULES:
127            raise Exception(f"test modules must not be listed: {name}")
128
129    return names
130
131
132def write_modules(fp, names):
133    print(f"// Auto-generated by {SCRIPT_NAME}.",
134          file=fp)
135    print("// List used to create sys.stdlib_module_names.", file=fp)
136    print(file=fp)
137    print("static const char* _Py_stdlib_module_names[] = {", file=fp)
138    for name in sorted(names):
139        print(f'"{name}",', file=fp)
140    print("};", file=fp)
141
142
143def main():
144    if not sysconfig.is_python_build():
145        print(f"ERROR: {sys.executable} is not a Python build",
146              file=sys.stderr)
147        sys.exit(1)
148
149    fp = sys.stdout
150    names = list_modules()
151    write_modules(fp, names)
152
153
154if __name__ == "__main__":
155    main()
156