1""" 2Generates a layout of Python for Windows from a build. 3 4See python make_layout.py --help for usage. 5""" 6 7__author__ = "Steve Dower <steve.dower@python.org>" 8__version__ = "3.8" 9 10import argparse 11import functools 12import os 13import re 14import shutil 15import subprocess 16import sys 17import tempfile 18import zipfile 19 20from pathlib import Path 21 22if __name__ == "__main__": 23 # Started directly, so enable relative imports 24 __path__ = [str(Path(__file__).resolve().parent)] 25 26from .support.appxmanifest import * 27from .support.catalog import * 28from .support.constants import * 29from .support.filesets import * 30from .support.logging import * 31from .support.options import * 32from .support.pip import * 33from .support.props import * 34from .support.nuspec import * 35 36TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") 37TEST_DIRS_ONLY = FileNameSet("test", "tests") 38 39IDLE_DIRS_ONLY = FileNameSet("idlelib") 40 41TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter") 42TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") 43TCLTK_FILES_ONLY = FileNameSet("turtle.py") 44 45VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") 46 47EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") 48EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") 49EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") 50EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") 51EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll") 52 53REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*") 54 55LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") 56 57PY_FILES = FileSuffixSet(".py") 58PYC_FILES = FileSuffixSet(".pyc") 59CAT_FILES = FileSuffixSet(".cat") 60CDF_FILES = FileSuffixSet(".cdf") 61 62DATA_DIRS = FileNameSet("data") 63 64TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser") 65TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt") 66 67 68def copy_if_modified(src, dest): 69 try: 70 dest_stat = os.stat(dest) 71 except FileNotFoundError: 72 do_copy = True 73 else: 74 src_stat = os.stat(src) 75 do_copy = ( 76 src_stat.st_mtime != dest_stat.st_mtime 77 or src_stat.st_size != dest_stat.st_size 78 ) 79 80 if do_copy: 81 shutil.copy2(src, dest) 82 83 84def get_lib_layout(ns): 85 def _c(f): 86 if f in EXCLUDE_FROM_LIB: 87 return False 88 if f.is_dir(): 89 if f in TEST_DIRS_ONLY: 90 return ns.include_tests 91 if f in TCLTK_DIRS_ONLY: 92 return ns.include_tcltk 93 if f in IDLE_DIRS_ONLY: 94 return ns.include_idle 95 if f in VENV_DIRS_ONLY: 96 return ns.include_venv 97 else: 98 if f in TCLTK_FILES_ONLY: 99 return ns.include_tcltk 100 return True 101 102 for dest, src in rglob(ns.source / "Lib", "**/*", _c): 103 yield dest, src 104 105 106def get_tcltk_lib(ns): 107 if not ns.include_tcltk: 108 return 109 110 tcl_lib = os.getenv("TCL_LIBRARY") 111 if not tcl_lib or not os.path.isdir(tcl_lib): 112 try: 113 with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f: 114 tcl_lib = f.read().strip() 115 except FileNotFoundError: 116 pass 117 if not tcl_lib or not os.path.isdir(tcl_lib): 118 log_warning("Failed to find TCL_LIBRARY") 119 return 120 121 for dest, src in rglob(Path(tcl_lib).parent, "**/*"): 122 yield "tcl/{}".format(dest), src 123 124 125def get_layout(ns): 126 def in_build(f, dest="", new_name=None): 127 n, _, x = f.rpartition(".") 128 n = new_name or n 129 src = ns.build / f 130 if ns.debug and src not in REQUIRED_DLLS: 131 if not src.stem.endswith("_d"): 132 src = src.parent / (src.stem + "_d" + src.suffix) 133 if not n.endswith("_d"): 134 n += "_d" 135 f = n + "." + x 136 yield dest + n + "." + x, src 137 if ns.include_symbols: 138 pdb = src.with_suffix(".pdb") 139 if pdb.is_file(): 140 yield dest + n + ".pdb", pdb 141 if ns.include_dev: 142 lib = src.with_suffix(".lib") 143 if lib.is_file(): 144 yield "libs/" + n + ".lib", lib 145 146 if ns.include_appxmanifest: 147 yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT)) 148 yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT)) 149 # For backwards compatibility, but we don't reference these ourselves. 150 yield from in_build("python_uwp.exe", new_name="python") 151 yield from in_build("pythonw_uwp.exe", new_name="pythonw") 152 else: 153 yield from in_build("python.exe", new_name="python") 154 yield from in_build("pythonw.exe", new_name="pythonw") 155 156 yield from in_build(PYTHON_DLL_NAME) 157 158 if ns.include_launchers and ns.include_appxmanifest: 159 if ns.include_pip: 160 yield from in_build("python_uwp.exe", new_name="pip{}".format(VER_DOT)) 161 if ns.include_idle: 162 yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT)) 163 164 if ns.include_stable: 165 yield from in_build(PYTHON_STABLE_DLL_NAME) 166 167 found_any = False 168 for dest, src in rglob(ns.build, "vcruntime*.dll"): 169 found_any = True 170 yield dest, src 171 if not found_any: 172 log_error("Failed to locate vcruntime DLL in the build.") 173 174 yield "LICENSE.txt", ns.build / "LICENSE.txt" 175 176 for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): 177 if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: 178 continue 179 if src in EXCLUDE_FROM_PYDS: 180 continue 181 if src in TEST_PYDS_ONLY and not ns.include_tests: 182 continue 183 if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: 184 continue 185 186 yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") 187 188 if ns.zip_lib: 189 zip_name = PYTHON_ZIP_NAME 190 yield zip_name, ns.temp / zip_name 191 else: 192 for dest, src in get_lib_layout(ns): 193 yield "Lib/{}".format(dest), src 194 195 if ns.include_venv: 196 yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") 197 yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") 198 199 if ns.include_tools: 200 201 def _c(d): 202 if d.is_dir(): 203 return d in TOOLS_DIRS 204 return d in TOOLS_FILES 205 206 for dest, src in rglob(ns.source / "Tools", "**/*", _c): 207 yield "Tools/{}".format(dest), src 208 209 if ns.include_underpth: 210 yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME 211 212 if ns.include_dev: 213 214 for dest, src in rglob(ns.source / "Include", "**/*.h"): 215 yield "include/{}".format(dest), src 216 src = ns.source / "PC" / "pyconfig.h" 217 yield "include/pyconfig.h", src 218 219 for dest, src in get_tcltk_lib(ns): 220 yield dest, src 221 222 if ns.include_pip: 223 for dest, src in get_pip_layout(ns): 224 if not isinstance(src, tuple) and ( 225 src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB 226 ): 227 continue 228 yield dest, src 229 230 if ns.include_chm: 231 for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME): 232 yield "Doc/{}".format(dest), src 233 234 if ns.include_html_doc: 235 for dest, src in rglob(ns.doc_build / "html", "**/*"): 236 yield "Doc/html/{}".format(dest), src 237 238 if ns.include_props: 239 for dest, src in get_props_layout(ns): 240 yield dest, src 241 242 if ns.include_nuspec: 243 for dest, src in get_nuspec_layout(ns): 244 yield dest, src 245 246 for dest, src in get_appx_layout(ns): 247 yield dest, src 248 249 if ns.include_cat: 250 if ns.flat_dlls: 251 yield ns.include_cat.name, ns.include_cat 252 else: 253 yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat 254 255 256def _compile_one_py(src, dest, name, optimize, checked=True): 257 import py_compile 258 259 if dest is not None: 260 dest = str(dest) 261 262 mode = ( 263 py_compile.PycInvalidationMode.CHECKED_HASH 264 if checked 265 else py_compile.PycInvalidationMode.UNCHECKED_HASH 266 ) 267 268 try: 269 return Path( 270 py_compile.compile( 271 str(src), 272 dest, 273 str(name), 274 doraise=True, 275 optimize=optimize, 276 invalidation_mode=mode, 277 ) 278 ) 279 except py_compile.PyCompileError: 280 log_warning("Failed to compile {}", src) 281 return None 282 283 284# name argument added to address bpo-37641 285def _py_temp_compile(src, name, ns, dest_dir=None, checked=True): 286 if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: 287 return None 288 dest = (dest_dir or ns.temp) / (src.stem + ".pyc") 289 return _compile_one_py(src, dest, name, optimize=2, checked=checked) 290 291 292def _write_to_zip(zf, dest, src, ns, checked=True): 293 pyc = _py_temp_compile(src, dest, ns, checked=checked) 294 if pyc: 295 try: 296 zf.write(str(pyc), dest.with_suffix(".pyc")) 297 finally: 298 try: 299 pyc.unlink() 300 except: 301 log_exception("Failed to delete {}", pyc) 302 return 303 304 if src in LIB2TO3_GRAMMAR_FILES: 305 from lib2to3.pgen2.driver import load_grammar 306 307 tmp = ns.temp / src.name 308 try: 309 shutil.copy(src, tmp) 310 load_grammar(str(tmp)) 311 for f in ns.temp.glob(src.stem + "*.pickle"): 312 zf.write(str(f), str(dest.parent / f.name)) 313 try: 314 f.unlink() 315 except: 316 log_exception("Failed to delete {}", f) 317 except: 318 log_exception("Failed to compile {}", src) 319 finally: 320 try: 321 tmp.unlink() 322 except: 323 log_exception("Failed to delete {}", tmp) 324 325 zf.write(str(src), str(dest)) 326 327 328def generate_source_files(ns): 329 if ns.zip_lib: 330 zip_name = PYTHON_ZIP_NAME 331 zip_path = ns.temp / zip_name 332 if zip_path.is_file(): 333 zip_path.unlink() 334 elif zip_path.is_dir(): 335 log_error( 336 "Cannot create zip file because a directory exists by the same name" 337 ) 338 return 339 log_info("Generating {} in {}", zip_name, ns.temp) 340 ns.temp.mkdir(parents=True, exist_ok=True) 341 with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: 342 for dest, src in get_lib_layout(ns): 343 _write_to_zip(zf, dest, src, ns, checked=False) 344 345 if ns.include_underpth: 346 log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp) 347 ns.temp.mkdir(parents=True, exist_ok=True) 348 with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f: 349 if ns.zip_lib: 350 print(PYTHON_ZIP_NAME, file=f) 351 if ns.include_pip: 352 print("packages", file=f) 353 else: 354 print("Lib", file=f) 355 print("Lib/site-packages", file=f) 356 if not ns.flat_dlls: 357 print("DLLs", file=f) 358 print(".", file=f) 359 print(file=f) 360 print("# Uncomment to run site.main() automatically", file=f) 361 print("#import site", file=f) 362 363 if ns.include_pip: 364 log_info("Extracting pip") 365 extract_pip_files(ns) 366 367 368def _create_zip_file(ns): 369 if not ns.zip: 370 return None 371 372 if ns.zip.is_file(): 373 try: 374 ns.zip.unlink() 375 except OSError: 376 log_exception("Unable to remove {}", ns.zip) 377 sys.exit(8) 378 elif ns.zip.is_dir(): 379 log_error("Cannot create ZIP file because {} is a directory", ns.zip) 380 sys.exit(8) 381 382 ns.zip.parent.mkdir(parents=True, exist_ok=True) 383 return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED) 384 385 386def copy_files(files, ns): 387 if ns.copy: 388 ns.copy.mkdir(parents=True, exist_ok=True) 389 390 try: 391 total = len(files) 392 except TypeError: 393 total = None 394 count = 0 395 396 zip_file = _create_zip_file(ns) 397 try: 398 need_compile = [] 399 in_catalog = [] 400 401 for dest, src in files: 402 count += 1 403 if count % 10 == 0: 404 if total: 405 log_info("Processed {:>4} of {} files", count, total) 406 else: 407 log_info("Processed {} files", count) 408 log_debug("Processing {!s}", src) 409 410 if isinstance(src, tuple): 411 src, content = src 412 if ns.copy: 413 log_debug("Copy {} -> {}", src, ns.copy / dest) 414 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) 415 with open(ns.copy / dest, "wb") as f: 416 f.write(content) 417 if ns.zip: 418 log_debug("Zip {} into {}", src, ns.zip) 419 zip_file.writestr(str(dest), content) 420 continue 421 422 if ( 423 ns.precompile 424 and src in PY_FILES 425 and src not in EXCLUDE_FROM_COMPILE 426 and src.parent not in DATA_DIRS 427 and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib")) 428 ): 429 if ns.copy: 430 need_compile.append((dest, ns.copy / dest)) 431 else: 432 (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True) 433 copy_if_modified(src, ns.temp / "Lib" / dest) 434 need_compile.append((dest, ns.temp / "Lib" / dest)) 435 436 if src not in EXCLUDE_FROM_CATALOG: 437 in_catalog.append((src.name, src)) 438 439 if ns.copy: 440 log_debug("Copy {} -> {}", src, ns.copy / dest) 441 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) 442 try: 443 copy_if_modified(src, ns.copy / dest) 444 except shutil.SameFileError: 445 pass 446 447 if ns.zip: 448 log_debug("Zip {} into {}", src, ns.zip) 449 zip_file.write(src, str(dest)) 450 451 if need_compile: 452 for dest, src in need_compile: 453 compiled = [ 454 _compile_one_py(src, None, dest, optimize=0), 455 _compile_one_py(src, None, dest, optimize=1), 456 _compile_one_py(src, None, dest, optimize=2), 457 ] 458 for c in compiled: 459 if not c: 460 continue 461 cdest = Path(dest).parent / Path(c).relative_to(src.parent) 462 if ns.zip: 463 log_debug("Zip {} into {}", c, ns.zip) 464 zip_file.write(c, str(cdest)) 465 in_catalog.append((cdest.name, cdest)) 466 467 if ns.catalog: 468 # Just write out the CDF now. Compilation and signing is 469 # an extra step 470 log_info("Generating {}", ns.catalog) 471 ns.catalog.parent.mkdir(parents=True, exist_ok=True) 472 write_catalog(ns.catalog, in_catalog) 473 474 finally: 475 if zip_file: 476 zip_file.close() 477 478 479def main(): 480 parser = argparse.ArgumentParser() 481 parser.add_argument("-v", help="Increase verbosity", action="count") 482 parser.add_argument( 483 "-s", 484 "--source", 485 metavar="dir", 486 help="The directory containing the repository root", 487 type=Path, 488 default=None, 489 ) 490 parser.add_argument( 491 "-b", "--build", metavar="dir", help="Specify the build directory", type=Path 492 ) 493 parser.add_argument( 494 "--arch", 495 metavar="architecture", 496 help="Specify the target architecture", 497 type=str, 498 default=None, 499 ) 500 parser.add_argument( 501 "--doc-build", 502 metavar="dir", 503 help="Specify the docs build directory", 504 type=Path, 505 default=None, 506 ) 507 parser.add_argument( 508 "--copy", 509 metavar="directory", 510 help="The name of the directory to copy an extracted layout to", 511 type=Path, 512 default=None, 513 ) 514 parser.add_argument( 515 "--zip", 516 metavar="file", 517 help="The ZIP file to write all files to", 518 type=Path, 519 default=None, 520 ) 521 parser.add_argument( 522 "--catalog", 523 metavar="file", 524 help="The CDF file to write catalog entries to", 525 type=Path, 526 default=None, 527 ) 528 parser.add_argument( 529 "--log", 530 metavar="file", 531 help="Write all operations to the specified file", 532 type=Path, 533 default=None, 534 ) 535 parser.add_argument( 536 "-t", 537 "--temp", 538 metavar="file", 539 help="A temporary working directory", 540 type=Path, 541 default=None, 542 ) 543 parser.add_argument( 544 "-d", "--debug", help="Include debug build", action="store_true" 545 ) 546 parser.add_argument( 547 "-p", 548 "--precompile", 549 help="Include .pyc files instead of .py", 550 action="store_true", 551 ) 552 parser.add_argument( 553 "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true" 554 ) 555 parser.add_argument( 556 "--flat-dlls", help="Does not create a DLLs directory", action="store_true" 557 ) 558 parser.add_argument( 559 "-a", 560 "--include-all", 561 help="Include all optional components", 562 action="store_true", 563 ) 564 parser.add_argument( 565 "--include-cat", 566 metavar="file", 567 help="Specify the catalog file to include", 568 type=Path, 569 default=None, 570 ) 571 for opt, help in get_argparse_options(): 572 parser.add_argument(opt, help=help, action="store_true") 573 574 ns = parser.parse_args() 575 update_presets(ns) 576 577 ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) 578 ns.build = ns.build or Path(sys.executable).parent 579 ns.temp = ns.temp or Path(tempfile.mkdtemp()) 580 ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") 581 if not ns.source.is_absolute(): 582 ns.source = (Path.cwd() / ns.source).resolve() 583 if not ns.build.is_absolute(): 584 ns.build = (Path.cwd() / ns.build).resolve() 585 if not ns.temp.is_absolute(): 586 ns.temp = (Path.cwd() / ns.temp).resolve() 587 if not ns.doc_build.is_absolute(): 588 ns.doc_build = (Path.cwd() / ns.doc_build).resolve() 589 if ns.include_cat and not ns.include_cat.is_absolute(): 590 ns.include_cat = (Path.cwd() / ns.include_cat).resolve() 591 if not ns.arch: 592 ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" 593 594 if ns.copy and not ns.copy.is_absolute(): 595 ns.copy = (Path.cwd() / ns.copy).resolve() 596 if ns.zip and not ns.zip.is_absolute(): 597 ns.zip = (Path.cwd() / ns.zip).resolve() 598 if ns.catalog and not ns.catalog.is_absolute(): 599 ns.catalog = (Path.cwd() / ns.catalog).resolve() 600 601 configure_logger(ns) 602 603 log_info( 604 """OPTIONS 605Source: {ns.source} 606Build: {ns.build} 607Temp: {ns.temp} 608Arch: {ns.arch} 609 610Copy to: {ns.copy} 611Zip to: {ns.zip} 612Catalog: {ns.catalog}""", 613 ns=ns, 614 ) 615 616 if ns.arch not in ("win32", "amd64", "arm32", "arm64"): 617 log_error("--arch is not a valid value (win32, amd64, arm32, arm64)") 618 return 4 619 if ns.arch in ("arm32", "arm64"): 620 for n in ("include_idle", "include_tcltk"): 621 if getattr(ns, n): 622 log_warning(f"Disabling --{n.replace('_', '-')} on unsupported platform") 623 setattr(ns, n, False) 624 625 if ns.include_idle and not ns.include_tcltk: 626 log_warning("Assuming --include-tcltk to support --include-idle") 627 ns.include_tcltk = True 628 629 try: 630 generate_source_files(ns) 631 files = list(get_layout(ns)) 632 copy_files(files, ns) 633 except KeyboardInterrupt: 634 log_info("Interrupted by Ctrl+C") 635 return 3 636 except SystemExit: 637 raise 638 except: 639 log_exception("Unhandled error") 640 641 if error_was_logged(): 642 log_error("Errors occurred.") 643 return 1 644 645 646if __name__ == "__main__": 647 sys.exit(int(main() or 0)) 648