1""" 2Generates a layout of Python for Windows from a build. 3 4See python make_layout.py --help for usage. 5""" 6 7__author__ = "Steve Dower <steve.dower@python.org>" 8__version__ = "3.8" 9 10import argparse 11import functools 12import os 13import re 14import shutil 15import subprocess 16import sys 17import tempfile 18import zipfile 19 20from pathlib import Path 21 22if __name__ == "__main__": 23 # Started directly, so enable relative imports 24 __path__ = [str(Path(__file__).resolve().parent)] 25 26from .support.appxmanifest import * 27from .support.catalog import * 28from .support.constants import * 29from .support.filesets import * 30from .support.logging import * 31from .support.options import * 32from .support.pip import * 33from .support.props import * 34from .support.nuspec import * 35 36BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py") 37BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py" 38 39TEST_PYDS_ONLY = FileStemSet("xxlimited", "_ctypes_test", "_test*") 40TEST_DIRS_ONLY = FileNameSet("test", "tests") 41 42IDLE_DIRS_ONLY = FileNameSet("idlelib") 43 44TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter") 45TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") 46TCLTK_FILES_ONLY = FileNameSet("turtle.py") 47 48VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") 49 50EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") 51EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") 52EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") 53EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") 54EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll") 55 56REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*") 57 58LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") 59 60PY_FILES = FileSuffixSet(".py") 61PYC_FILES = FileSuffixSet(".pyc") 62CAT_FILES = FileSuffixSet(".cat") 63CDF_FILES = FileSuffixSet(".cdf") 64 65DATA_DIRS = FileNameSet("data") 66 67TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser") 68TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt") 69 70 71def copy_if_modified(src, dest): 72 try: 73 dest_stat = os.stat(dest) 74 except FileNotFoundError: 75 do_copy = True 76 else: 77 src_stat = os.stat(src) 78 do_copy = ( 79 src_stat.st_mtime != dest_stat.st_mtime 80 or src_stat.st_size != dest_stat.st_size 81 ) 82 83 if do_copy: 84 shutil.copy2(src, dest) 85 86 87def get_lib_layout(ns): 88 def _c(f): 89 if f in EXCLUDE_FROM_LIB: 90 return False 91 if f.is_dir(): 92 if f in TEST_DIRS_ONLY: 93 return ns.include_tests 94 if f in TCLTK_DIRS_ONLY: 95 return ns.include_tcltk 96 if f in IDLE_DIRS_ONLY: 97 return ns.include_idle 98 if f in VENV_DIRS_ONLY: 99 return ns.include_venv 100 else: 101 if f in TCLTK_FILES_ONLY: 102 return ns.include_tcltk 103 if f in BDIST_WININST_FILES_ONLY: 104 return ns.include_bdist_wininst 105 return True 106 107 for dest, src in rglob(ns.source / "Lib", "**/*", _c): 108 yield dest, src 109 110 if not ns.include_bdist_wininst: 111 src = ns.source / BDIST_WININST_STUB 112 yield Path("distutils/command/bdist_wininst.py"), src 113 114 115def get_tcltk_lib(ns): 116 if not ns.include_tcltk: 117 return 118 119 tcl_lib = os.getenv("TCL_LIBRARY") 120 if not tcl_lib or not os.path.isdir(tcl_lib): 121 try: 122 with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f: 123 tcl_lib = f.read().strip() 124 except FileNotFoundError: 125 pass 126 if not tcl_lib or not os.path.isdir(tcl_lib): 127 log_warning("Failed to find TCL_LIBRARY") 128 return 129 130 for dest, src in rglob(Path(tcl_lib).parent, "**/*"): 131 yield "tcl/{}".format(dest), src 132 133 134def get_layout(ns): 135 def in_build(f, dest="", new_name=None): 136 n, _, x = f.rpartition(".") 137 n = new_name or n 138 src = ns.build / f 139 if ns.debug and src not in REQUIRED_DLLS: 140 if not src.stem.endswith("_d"): 141 src = src.parent / (src.stem + "_d" + src.suffix) 142 if not n.endswith("_d"): 143 n += "_d" 144 f = n + "." + x 145 yield dest + n + "." + x, src 146 if ns.include_symbols: 147 pdb = src.with_suffix(".pdb") 148 if pdb.is_file(): 149 yield dest + n + ".pdb", pdb 150 if ns.include_dev: 151 lib = src.with_suffix(".lib") 152 if lib.is_file(): 153 yield "libs/" + n + ".lib", lib 154 155 if ns.include_appxmanifest: 156 yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT)) 157 yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT)) 158 # For backwards compatibility, but we don't reference these ourselves. 159 yield from in_build("python_uwp.exe", new_name="python") 160 yield from in_build("pythonw_uwp.exe", new_name="pythonw") 161 else: 162 yield from in_build("python.exe", new_name="python") 163 yield from in_build("pythonw.exe", new_name="pythonw") 164 165 yield from in_build(PYTHON_DLL_NAME) 166 167 if ns.include_launchers and ns.include_appxmanifest: 168 if ns.include_pip: 169 yield from in_build("python_uwp.exe", new_name="pip{}".format(VER_DOT)) 170 if ns.include_idle: 171 yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT)) 172 173 if ns.include_stable: 174 yield from in_build(PYTHON_STABLE_DLL_NAME) 175 176 found_any = False 177 for dest, src in rglob(ns.build, "vcruntime*.dll"): 178 found_any = True 179 yield dest, src 180 if not found_any: 181 log_error("Failed to locate vcruntime DLL in the build.") 182 183 yield "LICENSE.txt", ns.build / "LICENSE.txt" 184 185 for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): 186 if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: 187 continue 188 if src in EXCLUDE_FROM_PYDS: 189 continue 190 if src in TEST_PYDS_ONLY and not ns.include_tests: 191 continue 192 if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: 193 continue 194 195 yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") 196 197 if ns.zip_lib: 198 zip_name = PYTHON_ZIP_NAME 199 yield zip_name, ns.temp / zip_name 200 else: 201 for dest, src in get_lib_layout(ns): 202 yield "Lib/{}".format(dest), src 203 204 if ns.include_venv: 205 yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") 206 yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") 207 208 if ns.include_tools: 209 210 def _c(d): 211 if d.is_dir(): 212 return d in TOOLS_DIRS 213 return d in TOOLS_FILES 214 215 for dest, src in rglob(ns.source / "Tools", "**/*", _c): 216 yield "Tools/{}".format(dest), src 217 218 if ns.include_underpth: 219 yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME 220 221 if ns.include_dev: 222 223 for dest, src in rglob(ns.source / "Include", "**/*.h"): 224 yield "include/{}".format(dest), src 225 src = ns.source / "PC" / "pyconfig.h" 226 yield "include/pyconfig.h", src 227 228 for dest, src in get_tcltk_lib(ns): 229 yield dest, src 230 231 if ns.include_pip: 232 for dest, src in get_pip_layout(ns): 233 if not isinstance(src, tuple) and ( 234 src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB 235 ): 236 continue 237 yield dest, src 238 239 if ns.include_chm: 240 for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME): 241 yield "Doc/{}".format(dest), src 242 243 if ns.include_html_doc: 244 for dest, src in rglob(ns.doc_build / "html", "**/*"): 245 yield "Doc/html/{}".format(dest), src 246 247 if ns.include_props: 248 for dest, src in get_props_layout(ns): 249 yield dest, src 250 251 if ns.include_nuspec: 252 for dest, src in get_nuspec_layout(ns): 253 yield dest, src 254 255 for dest, src in get_appx_layout(ns): 256 yield dest, src 257 258 if ns.include_cat: 259 if ns.flat_dlls: 260 yield ns.include_cat.name, ns.include_cat 261 else: 262 yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat 263 264 265def _compile_one_py(src, dest, name, optimize, checked=True): 266 import py_compile 267 268 if dest is not None: 269 dest = str(dest) 270 271 mode = ( 272 py_compile.PycInvalidationMode.CHECKED_HASH 273 if checked 274 else py_compile.PycInvalidationMode.UNCHECKED_HASH 275 ) 276 277 try: 278 return Path( 279 py_compile.compile( 280 str(src), 281 dest, 282 str(name), 283 doraise=True, 284 optimize=optimize, 285 invalidation_mode=mode, 286 ) 287 ) 288 except py_compile.PyCompileError: 289 log_warning("Failed to compile {}", src) 290 return None 291 292 293# name argument added to address bpo-37641 294def _py_temp_compile(src, name, ns, dest_dir=None, checked=True): 295 if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: 296 return None 297 dest = (dest_dir or ns.temp) / (src.stem + ".pyc") 298 return _compile_one_py(src, dest, name, optimize=2, checked=checked) 299 300 301def _write_to_zip(zf, dest, src, ns, checked=True): 302 pyc = _py_temp_compile(src, dest, ns, checked=checked) 303 if pyc: 304 try: 305 zf.write(str(pyc), dest.with_suffix(".pyc")) 306 finally: 307 try: 308 pyc.unlink() 309 except: 310 log_exception("Failed to delete {}", pyc) 311 return 312 313 if src in LIB2TO3_GRAMMAR_FILES: 314 from lib2to3.pgen2.driver import load_grammar 315 316 tmp = ns.temp / src.name 317 try: 318 shutil.copy(src, tmp) 319 load_grammar(str(tmp)) 320 for f in ns.temp.glob(src.stem + "*.pickle"): 321 zf.write(str(f), str(dest.parent / f.name)) 322 try: 323 f.unlink() 324 except: 325 log_exception("Failed to delete {}", f) 326 except: 327 log_exception("Failed to compile {}", src) 328 finally: 329 try: 330 tmp.unlink() 331 except: 332 log_exception("Failed to delete {}", tmp) 333 334 zf.write(str(src), str(dest)) 335 336 337def generate_source_files(ns): 338 if ns.zip_lib: 339 zip_name = PYTHON_ZIP_NAME 340 zip_path = ns.temp / zip_name 341 if zip_path.is_file(): 342 zip_path.unlink() 343 elif zip_path.is_dir(): 344 log_error( 345 "Cannot create zip file because a directory exists by the same name" 346 ) 347 return 348 log_info("Generating {} in {}", zip_name, ns.temp) 349 ns.temp.mkdir(parents=True, exist_ok=True) 350 with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: 351 for dest, src in get_lib_layout(ns): 352 _write_to_zip(zf, dest, src, ns, checked=False) 353 354 if ns.include_underpth: 355 log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp) 356 ns.temp.mkdir(parents=True, exist_ok=True) 357 with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f: 358 if ns.zip_lib: 359 print(PYTHON_ZIP_NAME, file=f) 360 if ns.include_pip: 361 print("packages", file=f) 362 else: 363 print("Lib", file=f) 364 print("Lib/site-packages", file=f) 365 if not ns.flat_dlls: 366 print("DLLs", file=f) 367 print(".", file=f) 368 print(file=f) 369 print("# Uncomment to run site.main() automatically", file=f) 370 print("#import site", file=f) 371 372 if ns.include_pip: 373 log_info("Extracting pip") 374 extract_pip_files(ns) 375 376 377def _create_zip_file(ns): 378 if not ns.zip: 379 return None 380 381 if ns.zip.is_file(): 382 try: 383 ns.zip.unlink() 384 except OSError: 385 log_exception("Unable to remove {}", ns.zip) 386 sys.exit(8) 387 elif ns.zip.is_dir(): 388 log_error("Cannot create ZIP file because {} is a directory", ns.zip) 389 sys.exit(8) 390 391 ns.zip.parent.mkdir(parents=True, exist_ok=True) 392 return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED) 393 394 395def copy_files(files, ns): 396 if ns.copy: 397 ns.copy.mkdir(parents=True, exist_ok=True) 398 399 try: 400 total = len(files) 401 except TypeError: 402 total = None 403 count = 0 404 405 zip_file = _create_zip_file(ns) 406 try: 407 need_compile = [] 408 in_catalog = [] 409 410 for dest, src in files: 411 count += 1 412 if count % 10 == 0: 413 if total: 414 log_info("Processed {:>4} of {} files", count, total) 415 else: 416 log_info("Processed {} files", count) 417 log_debug("Processing {!s}", src) 418 419 if isinstance(src, tuple): 420 src, content = src 421 if ns.copy: 422 log_debug("Copy {} -> {}", src, ns.copy / dest) 423 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) 424 with open(ns.copy / dest, "wb") as f: 425 f.write(content) 426 if ns.zip: 427 log_debug("Zip {} into {}", src, ns.zip) 428 zip_file.writestr(str(dest), content) 429 continue 430 431 if ( 432 ns.precompile 433 and src in PY_FILES 434 and src not in EXCLUDE_FROM_COMPILE 435 and src.parent not in DATA_DIRS 436 and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib")) 437 ): 438 if ns.copy: 439 need_compile.append((dest, ns.copy / dest)) 440 else: 441 (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True) 442 copy_if_modified(src, ns.temp / "Lib" / dest) 443 need_compile.append((dest, ns.temp / "Lib" / dest)) 444 445 if src not in EXCLUDE_FROM_CATALOG: 446 in_catalog.append((src.name, src)) 447 448 if ns.copy: 449 log_debug("Copy {} -> {}", src, ns.copy / dest) 450 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) 451 try: 452 copy_if_modified(src, ns.copy / dest) 453 except shutil.SameFileError: 454 pass 455 456 if ns.zip: 457 log_debug("Zip {} into {}", src, ns.zip) 458 zip_file.write(src, str(dest)) 459 460 if need_compile: 461 for dest, src in need_compile: 462 compiled = [ 463 _compile_one_py(src, None, dest, optimize=0), 464 _compile_one_py(src, None, dest, optimize=1), 465 _compile_one_py(src, None, dest, optimize=2), 466 ] 467 for c in compiled: 468 if not c: 469 continue 470 cdest = Path(dest).parent / Path(c).relative_to(src.parent) 471 if ns.zip: 472 log_debug("Zip {} into {}", c, ns.zip) 473 zip_file.write(c, str(cdest)) 474 in_catalog.append((cdest.name, cdest)) 475 476 if ns.catalog: 477 # Just write out the CDF now. Compilation and signing is 478 # an extra step 479 log_info("Generating {}", ns.catalog) 480 ns.catalog.parent.mkdir(parents=True, exist_ok=True) 481 write_catalog(ns.catalog, in_catalog) 482 483 finally: 484 if zip_file: 485 zip_file.close() 486 487 488def main(): 489 parser = argparse.ArgumentParser() 490 parser.add_argument("-v", help="Increase verbosity", action="count") 491 parser.add_argument( 492 "-s", 493 "--source", 494 metavar="dir", 495 help="The directory containing the repository root", 496 type=Path, 497 default=None, 498 ) 499 parser.add_argument( 500 "-b", "--build", metavar="dir", help="Specify the build directory", type=Path 501 ) 502 parser.add_argument( 503 "--arch", 504 metavar="architecture", 505 help="Specify the target architecture", 506 type=str, 507 default=None, 508 ) 509 parser.add_argument( 510 "--doc-build", 511 metavar="dir", 512 help="Specify the docs build directory", 513 type=Path, 514 default=None, 515 ) 516 parser.add_argument( 517 "--copy", 518 metavar="directory", 519 help="The name of the directory to copy an extracted layout to", 520 type=Path, 521 default=None, 522 ) 523 parser.add_argument( 524 "--zip", 525 metavar="file", 526 help="The ZIP file to write all files to", 527 type=Path, 528 default=None, 529 ) 530 parser.add_argument( 531 "--catalog", 532 metavar="file", 533 help="The CDF file to write catalog entries to", 534 type=Path, 535 default=None, 536 ) 537 parser.add_argument( 538 "--log", 539 metavar="file", 540 help="Write all operations to the specified file", 541 type=Path, 542 default=None, 543 ) 544 parser.add_argument( 545 "-t", 546 "--temp", 547 metavar="file", 548 help="A temporary working directory", 549 type=Path, 550 default=None, 551 ) 552 parser.add_argument( 553 "-d", "--debug", help="Include debug build", action="store_true" 554 ) 555 parser.add_argument( 556 "-p", 557 "--precompile", 558 help="Include .pyc files instead of .py", 559 action="store_true", 560 ) 561 parser.add_argument( 562 "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true" 563 ) 564 parser.add_argument( 565 "--flat-dlls", help="Does not create a DLLs directory", action="store_true" 566 ) 567 parser.add_argument( 568 "-a", 569 "--include-all", 570 help="Include all optional components", 571 action="store_true", 572 ) 573 parser.add_argument( 574 "--include-cat", 575 metavar="file", 576 help="Specify the catalog file to include", 577 type=Path, 578 default=None, 579 ) 580 for opt, help in get_argparse_options(): 581 parser.add_argument(opt, help=help, action="store_true") 582 583 ns = parser.parse_args() 584 update_presets(ns) 585 586 ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) 587 ns.build = ns.build or Path(sys.executable).parent 588 ns.temp = ns.temp or Path(tempfile.mkdtemp()) 589 ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") 590 if not ns.source.is_absolute(): 591 ns.source = (Path.cwd() / ns.source).resolve() 592 if not ns.build.is_absolute(): 593 ns.build = (Path.cwd() / ns.build).resolve() 594 if not ns.temp.is_absolute(): 595 ns.temp = (Path.cwd() / ns.temp).resolve() 596 if not ns.doc_build.is_absolute(): 597 ns.doc_build = (Path.cwd() / ns.doc_build).resolve() 598 if ns.include_cat and not ns.include_cat.is_absolute(): 599 ns.include_cat = (Path.cwd() / ns.include_cat).resolve() 600 if not ns.arch: 601 ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" 602 603 if ns.copy and not ns.copy.is_absolute(): 604 ns.copy = (Path.cwd() / ns.copy).resolve() 605 if ns.zip and not ns.zip.is_absolute(): 606 ns.zip = (Path.cwd() / ns.zip).resolve() 607 if ns.catalog and not ns.catalog.is_absolute(): 608 ns.catalog = (Path.cwd() / ns.catalog).resolve() 609 610 configure_logger(ns) 611 612 log_info( 613 """OPTIONS 614Source: {ns.source} 615Build: {ns.build} 616Temp: {ns.temp} 617Arch: {ns.arch} 618 619Copy to: {ns.copy} 620Zip to: {ns.zip} 621Catalog: {ns.catalog}""", 622 ns=ns, 623 ) 624 625 if ns.arch not in ("win32", "amd64", "arm32", "arm64"): 626 log_error("--arch is not a valid value (win32, amd64, arm32, arm64)") 627 return 4 628 if ns.arch in ("arm32", "arm64"): 629 for n in ("include_idle", "include_tcltk"): 630 if getattr(ns, n): 631 log_warning(f"Disabling --{n.replace('_', '-')} on unsupported platform") 632 setattr(ns, n, False) 633 634 if ns.include_idle and not ns.include_tcltk: 635 log_warning("Assuming --include-tcltk to support --include-idle") 636 ns.include_tcltk = True 637 638 try: 639 generate_source_files(ns) 640 files = list(get_layout(ns)) 641 copy_files(files, ns) 642 except KeyboardInterrupt: 643 log_info("Interrupted by Ctrl+C") 644 return 3 645 except SystemExit: 646 raise 647 except: 648 log_exception("Unhandled error") 649 650 if error_was_logged(): 651 log_error("Errors occurred.") 652 return 1 653 654 655if __name__ == "__main__": 656 sys.exit(int(main() or 0)) 657