• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# This script lists the names of standard library modules
2# to update Python/stdlib_mod_names.h
3import os.path
4import re
5import subprocess
6import sys
7import sysconfig
8
9
10SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
11STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
12MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup')
13SETUP_PY = os.path.join(SRC_DIR, 'setup.py')
14TEST_EMBED = os.path.join(SRC_DIR, 'Programs', '_testembed')
15
16IGNORE = {
17    '__init__',
18    '__pycache__',
19    'site-packages',
20
21    # Test modules and packages
22    '__hello__',
23    '__phello__',
24    '_ctypes_test',
25    '_testbuffer',
26    '_testcapi',
27    '_testconsole',
28    '_testimportmultiple',
29    '_testinternalcapi',
30    '_testmultiphase',
31    '_xxsubinterpreters',
32    '_xxtestfuzz',
33    'distutils.tests',
34    'idlelib.idle_test',
35    'lib2to3.tests',
36    'test',
37    'xxlimited',
38    'xxlimited_35',
39    'xxsubtype',
40}
41
42# Windows extension modules
43WINDOWS_MODULES = (
44    '_msi',
45    '_overlapped',
46    '_testconsole',
47    '_winapi',
48    'msvcrt',
49    'nt',
50    'winreg',
51    'winsound'
52)
53
54# macOS extension modules
55MACOS_MODULES = (
56    '_scproxy',
57)
58
59# Pure Python modules (Lib/*.py)
60def list_python_modules(names):
61    for filename in os.listdir(STDLIB_PATH):
62        if not filename.endswith(".py"):
63            continue
64        name = filename.removesuffix(".py")
65        names.add(name)
66
67
68# Packages in Lib/
69def list_packages(names):
70    for name in os.listdir(STDLIB_PATH):
71        if name in IGNORE:
72            continue
73        package_path = os.path.join(STDLIB_PATH, name)
74        if not os.path.isdir(package_path):
75            continue
76        if any(package_file.endswith(".py")
77               for package_file in os.listdir(package_path)):
78            names.add(name)
79
80
81# Extension modules built by setup.py
82def list_setup_extensions(names):
83    cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"]
84    output = subprocess.check_output(cmd)
85    output = output.decode("utf8")
86    extensions = output.splitlines()
87    names |= set(extensions)
88
89
90# Built-in and extension modules built by Modules/Setup
91def list_modules_setup_extensions(names):
92    assign_var = re.compile("^[A-Z]+=")
93
94    with open(MODULES_SETUP, encoding="utf-8") as modules_fp:
95        for line in modules_fp:
96            # Strip comment
97            line = line.partition("#")[0]
98            line = line.rstrip()
99            if not line:
100                continue
101            if assign_var.match(line):
102                # Ignore "VAR=VALUE"
103                continue
104            if line in ("*disabled*", "*shared*"):
105                continue
106            parts = line.split()
107            if len(parts) < 2:
108                continue
109            # "errno errnomodule.c" => write "errno"
110            name = parts[0]
111            names.add(name)
112
113
114# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
115# Use the "./Programs/_testembed list_frozen" command.
116def list_frozen(names):
117    args = [TEST_EMBED, 'list_frozen']
118    proc = subprocess.run(args, stdout=subprocess.PIPE, text=True)
119    exitcode = proc.returncode
120    if exitcode:
121        cmd = ' '.join(args)
122        print(f"{cmd} failed with exitcode {exitcode}")
123        sys.exit(exitcode)
124    for line in proc.stdout.splitlines():
125        name = line.strip()
126        names.add(name)
127
128
129def list_modules():
130    names = set(sys.builtin_module_names) | set(WINDOWS_MODULES) | set(MACOS_MODULES)
131    list_modules_setup_extensions(names)
132    list_setup_extensions(names)
133    list_packages(names)
134    list_python_modules(names)
135    list_frozen(names)
136
137    # Remove ignored packages and modules
138    for name in list(names):
139        package_name = name.split('.')[0]
140        # package_name can be equal to name
141        if package_name in IGNORE:
142            names.discard(name)
143
144    for name in names:
145        if "." in name:
146            raise Exception("sub-modules must not be listed")
147
148    return names
149
150
151def write_modules(fp, names):
152    print("// Auto-generated by Tools/scripts/generate_stdlib_module_names.py.",
153          file=fp)
154    print("// List used to create sys.stdlib_module_names.", file=fp)
155    print(file=fp)
156    print("static const char* _Py_stdlib_module_names[] = {", file=fp)
157    for name in sorted(names):
158        print(f'"{name}",', file=fp)
159    print("};", file=fp)
160
161
162def main():
163    if not sysconfig.is_python_build():
164        print(f"ERROR: {sys.executable} is not a Python build",
165              file=sys.stderr)
166        sys.exit(1)
167
168    fp = sys.stdout
169    names = list_modules()
170    write_modules(fp, names)
171
172
173if __name__ == "__main__":
174    main()
175