1""" 2Generates a layout of Python for Windows from a build. 3 4See python make_layout.py --help for usage. 5""" 6 7__author__ = "Steve Dower <steve.dower@python.org>" 8__version__ = "3.8" 9 10import argparse 11import functools 12import os 13import re 14import shutil 15import subprocess 16import sys 17import tempfile 18import zipfile 19 20from pathlib import Path 21 22if __name__ == "__main__": 23 # Started directly, so enable relative imports 24 __path__ = [str(Path(__file__).resolve().parent)] 25 26from .support.appxmanifest import * 27from .support.catalog import * 28from .support.constants import * 29from .support.filesets import * 30from .support.logging import * 31from .support.options import * 32from .support.pip import * 33from .support.props import * 34from .support.nuspec import * 35 36BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py") 37BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py" 38 39TEST_PYDS_ONLY = FileStemSet("xxlimited", "_ctypes_test", "_test*") 40TEST_DIRS_ONLY = FileNameSet("test", "tests") 41 42IDLE_DIRS_ONLY = FileNameSet("idlelib") 43 44TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter") 45TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") 46TCLTK_FILES_ONLY = FileNameSet("turtle.py") 47 48VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") 49 50EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") 51EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") 52EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") 53EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") 54EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll") 55 56REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*") 57 58LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") 59 60PY_FILES = FileSuffixSet(".py") 61PYC_FILES = FileSuffixSet(".pyc") 62CAT_FILES = FileSuffixSet(".cat") 63CDF_FILES = FileSuffixSet(".cdf") 64 65DATA_DIRS = FileNameSet("data") 66 67TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser") 68TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt") 69 70 71def copy_if_modified(src, dest): 72 try: 73 dest_stat = os.stat(dest) 74 except FileNotFoundError: 75 do_copy = True 76 else: 77 src_stat = os.stat(src) 78 do_copy = ( 79 src_stat.st_mtime != dest_stat.st_mtime 80 or src_stat.st_size != dest_stat.st_size 81 ) 82 83 if do_copy: 84 shutil.copy2(src, dest) 85 86 87def get_lib_layout(ns): 88 def _c(f): 89 if f in EXCLUDE_FROM_LIB: 90 return False 91 if f.is_dir(): 92 if f in TEST_DIRS_ONLY: 93 return ns.include_tests 94 if f in TCLTK_DIRS_ONLY: 95 return ns.include_tcltk 96 if f in IDLE_DIRS_ONLY: 97 return ns.include_idle 98 if f in VENV_DIRS_ONLY: 99 return ns.include_venv 100 else: 101 if f in TCLTK_FILES_ONLY: 102 return ns.include_tcltk 103 if f in BDIST_WININST_FILES_ONLY: 104 return ns.include_bdist_wininst 105 return True 106 107 for dest, src in rglob(ns.source / "Lib", "**/*", _c): 108 yield dest, src 109 110 if not ns.include_bdist_wininst: 111 src = ns.source / BDIST_WININST_STUB 112 yield Path("distutils/command/bdist_wininst.py"), src 113 114 115def get_tcltk_lib(ns): 116 if not ns.include_tcltk: 117 return 118 119 tcl_lib = os.getenv("TCL_LIBRARY") 120 if not tcl_lib or not os.path.isdir(tcl_lib): 121 try: 122 with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f: 123 tcl_lib = f.read().strip() 124 except FileNotFoundError: 125 pass 126 if not tcl_lib or not os.path.isdir(tcl_lib): 127 log_warning("Failed to find TCL_LIBRARY") 128 return 129 130 for dest, src in rglob(Path(tcl_lib).parent, "**/*"): 131 yield "tcl/{}".format(dest), src 132 133 134def get_layout(ns): 135 def in_build(f, dest="", new_name=None): 136 n, _, x = f.rpartition(".") 137 n = new_name or n 138 src = ns.build / f 139 if ns.debug and src not in REQUIRED_DLLS: 140 if not src.stem.endswith("_d"): 141 src = src.parent / (src.stem + "_d" + src.suffix) 142 if not n.endswith("_d"): 143 n += "_d" 144 f = n + "." + x 145 yield dest + n + "." + x, src 146 if ns.include_symbols: 147 pdb = src.with_suffix(".pdb") 148 if pdb.is_file(): 149 yield dest + n + ".pdb", pdb 150 if ns.include_dev: 151 lib = src.with_suffix(".lib") 152 if lib.is_file(): 153 yield "libs/" + n + ".lib", lib 154 155 if ns.include_appxmanifest: 156 yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT)) 157 yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT)) 158 # For backwards compatibility, but we don't reference these ourselves. 159 yield from in_build("python_uwp.exe", new_name="python") 160 yield from in_build("pythonw_uwp.exe", new_name="pythonw") 161 else: 162 yield from in_build("python.exe", new_name="python") 163 yield from in_build("pythonw.exe", new_name="pythonw") 164 165 yield from in_build(PYTHON_DLL_NAME) 166 167 if ns.include_launchers and ns.include_appxmanifest: 168 if ns.include_pip: 169 yield from in_build("python_uwp.exe", new_name="pip{}".format(VER_DOT)) 170 if ns.include_idle: 171 yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT)) 172 173 if ns.include_stable: 174 yield from in_build(PYTHON_STABLE_DLL_NAME) 175 176 for dest, src in rglob(ns.build, "vcruntime*.dll"): 177 yield dest, src 178 179 yield "LICENSE.txt", ns.build / "LICENSE.txt" 180 181 for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): 182 if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: 183 continue 184 if src in EXCLUDE_FROM_PYDS: 185 continue 186 if src in TEST_PYDS_ONLY and not ns.include_tests: 187 continue 188 if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: 189 continue 190 191 yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") 192 193 if ns.zip_lib: 194 zip_name = PYTHON_ZIP_NAME 195 yield zip_name, ns.temp / zip_name 196 else: 197 for dest, src in get_lib_layout(ns): 198 yield "Lib/{}".format(dest), src 199 200 if ns.include_venv: 201 yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") 202 yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") 203 204 if ns.include_tools: 205 206 def _c(d): 207 if d.is_dir(): 208 return d in TOOLS_DIRS 209 return d in TOOLS_FILES 210 211 for dest, src in rglob(ns.source / "Tools", "**/*", _c): 212 yield "Tools/{}".format(dest), src 213 214 if ns.include_underpth: 215 yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME 216 217 if ns.include_dev: 218 219 for dest, src in rglob(ns.source / "Include", "**/*.h"): 220 yield "include/{}".format(dest), src 221 src = ns.source / "PC" / "pyconfig.h" 222 yield "include/pyconfig.h", src 223 224 for dest, src in get_tcltk_lib(ns): 225 yield dest, src 226 227 if ns.include_pip: 228 for dest, src in get_pip_layout(ns): 229 if not isinstance(src, tuple) and ( 230 src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB 231 ): 232 continue 233 yield dest, src 234 235 if ns.include_chm: 236 for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME): 237 yield "Doc/{}".format(dest), src 238 239 if ns.include_html_doc: 240 for dest, src in rglob(ns.doc_build / "html", "**/*"): 241 yield "Doc/html/{}".format(dest), src 242 243 if ns.include_props: 244 for dest, src in get_props_layout(ns): 245 yield dest, src 246 247 if ns.include_nuspec: 248 for dest, src in get_nuspec_layout(ns): 249 yield dest, src 250 251 for dest, src in get_appx_layout(ns): 252 yield dest, src 253 254 if ns.include_cat: 255 if ns.flat_dlls: 256 yield ns.include_cat.name, ns.include_cat 257 else: 258 yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat 259 260 261def _compile_one_py(src, dest, name, optimize, checked=True): 262 import py_compile 263 264 if dest is not None: 265 dest = str(dest) 266 267 mode = ( 268 py_compile.PycInvalidationMode.CHECKED_HASH 269 if checked 270 else py_compile.PycInvalidationMode.UNCHECKED_HASH 271 ) 272 273 try: 274 return Path( 275 py_compile.compile( 276 str(src), 277 dest, 278 str(name), 279 doraise=True, 280 optimize=optimize, 281 invalidation_mode=mode, 282 ) 283 ) 284 except py_compile.PyCompileError: 285 log_warning("Failed to compile {}", src) 286 return None 287 288 289# name argument added to address bpo-37641 290def _py_temp_compile(src, name, ns, dest_dir=None, checked=True): 291 if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: 292 return None 293 dest = (dest_dir or ns.temp) / (src.stem + ".pyc") 294 return _compile_one_py(src, dest, name, optimize=2, checked=checked) 295 296 297def _write_to_zip(zf, dest, src, ns, checked=True): 298 pyc = _py_temp_compile(src, dest, ns, checked=checked) 299 if pyc: 300 try: 301 zf.write(str(pyc), dest.with_suffix(".pyc")) 302 finally: 303 try: 304 pyc.unlink() 305 except: 306 log_exception("Failed to delete {}", pyc) 307 return 308 309 if src in LIB2TO3_GRAMMAR_FILES: 310 from lib2to3.pgen2.driver import load_grammar 311 312 tmp = ns.temp / src.name 313 try: 314 shutil.copy(src, tmp) 315 load_grammar(str(tmp)) 316 for f in ns.temp.glob(src.stem + "*.pickle"): 317 zf.write(str(f), str(dest.parent / f.name)) 318 try: 319 f.unlink() 320 except: 321 log_exception("Failed to delete {}", f) 322 except: 323 log_exception("Failed to compile {}", src) 324 finally: 325 try: 326 tmp.unlink() 327 except: 328 log_exception("Failed to delete {}", tmp) 329 330 zf.write(str(src), str(dest)) 331 332 333def generate_source_files(ns): 334 if ns.zip_lib: 335 zip_name = PYTHON_ZIP_NAME 336 zip_path = ns.temp / zip_name 337 if zip_path.is_file(): 338 zip_path.unlink() 339 elif zip_path.is_dir(): 340 log_error( 341 "Cannot create zip file because a directory exists by the same name" 342 ) 343 return 344 log_info("Generating {} in {}", zip_name, ns.temp) 345 ns.temp.mkdir(parents=True, exist_ok=True) 346 with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: 347 for dest, src in get_lib_layout(ns): 348 _write_to_zip(zf, dest, src, ns, checked=False) 349 350 if ns.include_underpth: 351 log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp) 352 ns.temp.mkdir(parents=True, exist_ok=True) 353 with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f: 354 if ns.zip_lib: 355 print(PYTHON_ZIP_NAME, file=f) 356 if ns.include_pip: 357 print("packages", file=f) 358 else: 359 print("Lib", file=f) 360 print("Lib/site-packages", file=f) 361 if not ns.flat_dlls: 362 print("DLLs", file=f) 363 print(".", file=f) 364 print(file=f) 365 print("# Uncomment to run site.main() automatically", file=f) 366 print("#import site", file=f) 367 368 if ns.include_pip: 369 log_info("Extracting pip") 370 extract_pip_files(ns) 371 372 373def _create_zip_file(ns): 374 if not ns.zip: 375 return None 376 377 if ns.zip.is_file(): 378 try: 379 ns.zip.unlink() 380 except OSError: 381 log_exception("Unable to remove {}", ns.zip) 382 sys.exit(8) 383 elif ns.zip.is_dir(): 384 log_error("Cannot create ZIP file because {} is a directory", ns.zip) 385 sys.exit(8) 386 387 ns.zip.parent.mkdir(parents=True, exist_ok=True) 388 return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED) 389 390 391def copy_files(files, ns): 392 if ns.copy: 393 ns.copy.mkdir(parents=True, exist_ok=True) 394 395 try: 396 total = len(files) 397 except TypeError: 398 total = None 399 count = 0 400 401 zip_file = _create_zip_file(ns) 402 try: 403 need_compile = [] 404 in_catalog = [] 405 406 for dest, src in files: 407 count += 1 408 if count % 10 == 0: 409 if total: 410 log_info("Processed {:>4} of {} files", count, total) 411 else: 412 log_info("Processed {} files", count) 413 log_debug("Processing {!s}", src) 414 415 if isinstance(src, tuple): 416 src, content = src 417 if ns.copy: 418 log_debug("Copy {} -> {}", src, ns.copy / dest) 419 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) 420 with open(ns.copy / dest, "wb") as f: 421 f.write(content) 422 if ns.zip: 423 log_debug("Zip {} into {}", src, ns.zip) 424 zip_file.writestr(str(dest), content) 425 continue 426 427 if ( 428 ns.precompile 429 and src in PY_FILES 430 and src not in EXCLUDE_FROM_COMPILE 431 and src.parent not in DATA_DIRS 432 and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib")) 433 ): 434 if ns.copy: 435 need_compile.append((dest, ns.copy / dest)) 436 else: 437 (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True) 438 copy_if_modified(src, ns.temp / "Lib" / dest) 439 need_compile.append((dest, ns.temp / "Lib" / dest)) 440 441 if src not in EXCLUDE_FROM_CATALOG: 442 in_catalog.append((src.name, src)) 443 444 if ns.copy: 445 log_debug("Copy {} -> {}", src, ns.copy / dest) 446 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) 447 try: 448 copy_if_modified(src, ns.copy / dest) 449 except shutil.SameFileError: 450 pass 451 452 if ns.zip: 453 log_debug("Zip {} into {}", src, ns.zip) 454 zip_file.write(src, str(dest)) 455 456 if need_compile: 457 for dest, src in need_compile: 458 compiled = [ 459 _compile_one_py(src, None, dest, optimize=0), 460 _compile_one_py(src, None, dest, optimize=1), 461 _compile_one_py(src, None, dest, optimize=2), 462 ] 463 for c in compiled: 464 if not c: 465 continue 466 cdest = Path(dest).parent / Path(c).relative_to(src.parent) 467 if ns.zip: 468 log_debug("Zip {} into {}", c, ns.zip) 469 zip_file.write(c, str(cdest)) 470 in_catalog.append((cdest.name, cdest)) 471 472 if ns.catalog: 473 # Just write out the CDF now. Compilation and signing is 474 # an extra step 475 log_info("Generating {}", ns.catalog) 476 ns.catalog.parent.mkdir(parents=True, exist_ok=True) 477 write_catalog(ns.catalog, in_catalog) 478 479 finally: 480 if zip_file: 481 zip_file.close() 482 483 484def main(): 485 parser = argparse.ArgumentParser() 486 parser.add_argument("-v", help="Increase verbosity", action="count") 487 parser.add_argument( 488 "-s", 489 "--source", 490 metavar="dir", 491 help="The directory containing the repository root", 492 type=Path, 493 default=None, 494 ) 495 parser.add_argument( 496 "-b", "--build", metavar="dir", help="Specify the build directory", type=Path 497 ) 498 parser.add_argument( 499 "--arch", 500 metavar="architecture", 501 help="Specify the target architecture", 502 type=str, 503 default=None, 504 ) 505 parser.add_argument( 506 "--doc-build", 507 metavar="dir", 508 help="Specify the docs build directory", 509 type=Path, 510 default=None, 511 ) 512 parser.add_argument( 513 "--copy", 514 metavar="directory", 515 help="The name of the directory to copy an extracted layout to", 516 type=Path, 517 default=None, 518 ) 519 parser.add_argument( 520 "--zip", 521 metavar="file", 522 help="The ZIP file to write all files to", 523 type=Path, 524 default=None, 525 ) 526 parser.add_argument( 527 "--catalog", 528 metavar="file", 529 help="The CDF file to write catalog entries to", 530 type=Path, 531 default=None, 532 ) 533 parser.add_argument( 534 "--log", 535 metavar="file", 536 help="Write all operations to the specified file", 537 type=Path, 538 default=None, 539 ) 540 parser.add_argument( 541 "-t", 542 "--temp", 543 metavar="file", 544 help="A temporary working directory", 545 type=Path, 546 default=None, 547 ) 548 parser.add_argument( 549 "-d", "--debug", help="Include debug build", action="store_true" 550 ) 551 parser.add_argument( 552 "-p", 553 "--precompile", 554 help="Include .pyc files instead of .py", 555 action="store_true", 556 ) 557 parser.add_argument( 558 "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true" 559 ) 560 parser.add_argument( 561 "--flat-dlls", help="Does not create a DLLs directory", action="store_true" 562 ) 563 parser.add_argument( 564 "-a", 565 "--include-all", 566 help="Include all optional components", 567 action="store_true", 568 ) 569 parser.add_argument( 570 "--include-cat", 571 metavar="file", 572 help="Specify the catalog file to include", 573 type=Path, 574 default=None, 575 ) 576 for opt, help in get_argparse_options(): 577 parser.add_argument(opt, help=help, action="store_true") 578 579 ns = parser.parse_args() 580 update_presets(ns) 581 582 ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) 583 ns.build = ns.build or Path(sys.executable).parent 584 ns.temp = ns.temp or Path(tempfile.mkdtemp()) 585 ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") 586 if not ns.source.is_absolute(): 587 ns.source = (Path.cwd() / ns.source).resolve() 588 if not ns.build.is_absolute(): 589 ns.build = (Path.cwd() / ns.build).resolve() 590 if not ns.temp.is_absolute(): 591 ns.temp = (Path.cwd() / ns.temp).resolve() 592 if not ns.doc_build.is_absolute(): 593 ns.doc_build = (Path.cwd() / ns.doc_build).resolve() 594 if ns.include_cat and not ns.include_cat.is_absolute(): 595 ns.include_cat = (Path.cwd() / ns.include_cat).resolve() 596 if not ns.arch: 597 ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" 598 599 if ns.copy and not ns.copy.is_absolute(): 600 ns.copy = (Path.cwd() / ns.copy).resolve() 601 if ns.zip and not ns.zip.is_absolute(): 602 ns.zip = (Path.cwd() / ns.zip).resolve() 603 if ns.catalog and not ns.catalog.is_absolute(): 604 ns.catalog = (Path.cwd() / ns.catalog).resolve() 605 606 configure_logger(ns) 607 608 log_info( 609 """OPTIONS 610Source: {ns.source} 611Build: {ns.build} 612Temp: {ns.temp} 613Arch: {ns.arch} 614 615Copy to: {ns.copy} 616Zip to: {ns.zip} 617Catalog: {ns.catalog}""", 618 ns=ns, 619 ) 620 621 if ns.arch not in ("win32", "amd64", "arm32", "arm64"): 622 log_error("--arch is not a valid value (win32, amd64, arm32, arm64)") 623 return 4 624 if ns.arch in ("arm32", "arm64"): 625 for n in ("include_idle", "include_tcltk"): 626 if getattr(ns, n): 627 log_warning(f"Disabling --{n.replace('_', '-')} on unsupported platform") 628 setattr(ns, n, False) 629 630 if ns.include_idle and not ns.include_tcltk: 631 log_warning("Assuming --include-tcltk to support --include-idle") 632 ns.include_tcltk = True 633 634 try: 635 generate_source_files(ns) 636 files = list(get_layout(ns)) 637 copy_files(files, ns) 638 except KeyboardInterrupt: 639 log_info("Interrupted by Ctrl+C") 640 return 3 641 except SystemExit: 642 raise 643 except: 644 log_exception("Unhandled error") 645 646 if error_was_logged(): 647 log_error("Errors occurred.") 648 return 1 649 650 651if __name__ == "__main__": 652 sys.exit(int(main() or 0)) 653