1"""Module/script to byte-compile all .py files to .pyc files. 2 3When called as a script with arguments, this compiles the directories 4given as arguments recursively; the -l option prevents it from 5recursing into directories. 6 7Without arguments, if compiles all modules on sys.path, without 8recursing into subdirectories. (Even though it should do so for 9packages -- for now, you'll have to deal with packages separately.) 10 11See module py_compile for details of the actual byte-compilation. 12""" 13import os 14import sys 15import importlib.util 16import py_compile 17import struct 18import filecmp 19 20from functools import partial 21from pathlib import Path 22 23__all__ = ["compile_dir","compile_file","compile_path"] 24 25def _walk_dir(dir, maxlevels, quiet=0): 26 if quiet < 2 and isinstance(dir, os.PathLike): 27 dir = os.fspath(dir) 28 if not quiet: 29 print('Listing {!r}...'.format(dir)) 30 try: 31 names = os.listdir(dir) 32 except OSError: 33 if quiet < 2: 34 print("Can't list {!r}".format(dir)) 35 names = [] 36 names.sort() 37 for name in names: 38 if name == '__pycache__': 39 continue 40 fullname = os.path.join(dir, name) 41 if not os.path.isdir(fullname): 42 yield fullname 43 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and 44 os.path.isdir(fullname) and not os.path.islink(fullname)): 45 yield from _walk_dir(fullname, maxlevels=maxlevels - 1, 46 quiet=quiet) 47 48def compile_dir(dir, maxlevels=None, ddir=None, force=False, 49 rx=None, quiet=0, legacy=False, optimize=-1, workers=1, 50 invalidation_mode=None, *, stripdir=None, 51 prependdir=None, limit_sl_dest=None, hardlink_dupes=False): 52 """Byte-compile all modules in the given directory tree. 53 54 Arguments (only dir is required): 55 56 dir: the directory to byte-compile 57 maxlevels: maximum recursion level (default `sys.getrecursionlimit()`) 58 ddir: the directory that will be prepended to the path to the 59 file as it is compiled into each byte-code file. 60 force: if True, force compilation, even if timestamps are up-to-date 61 quiet: full output with False or 0, errors only with 1, 62 no output with 2 63 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 64 optimize: int or list of optimization levels or -1 for level of 65 the interpreter. Multiple levels leads to multiple compiled 66 files each with one optimization level. 67 workers: maximum number of parallel workers 68 invalidation_mode: how the up-to-dateness of the pyc will be checked 69 stripdir: part of path to left-strip from source file path 70 prependdir: path to prepend to beginning of original file path, applied 71 after stripdir 72 limit_sl_dest: ignore symlinks if they are pointing outside of 73 the defined path 74 hardlink_dupes: hardlink duplicated pyc files 75 """ 76 ProcessPoolExecutor = None 77 if ddir is not None and (stripdir is not None or prependdir is not None): 78 raise ValueError(("Destination dir (ddir) cannot be used " 79 "in combination with stripdir or prependdir")) 80 if ddir is not None: 81 stripdir = dir 82 prependdir = ddir 83 ddir = None 84 if workers < 0: 85 raise ValueError('workers must be greater or equal to 0') 86 if workers != 1: 87 try: 88 # Only import when needed, as low resource platforms may 89 # fail to import it 90 from concurrent.futures import ProcessPoolExecutor 91 except ImportError: 92 workers = 1 93 if maxlevels is None: 94 maxlevels = sys.getrecursionlimit() 95 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) 96 success = True 97 if workers != 1 and ProcessPoolExecutor is not None: 98 # If workers == 0, let ProcessPoolExecutor choose 99 workers = workers or None 100 with ProcessPoolExecutor(max_workers=workers) as executor: 101 results = executor.map(partial(compile_file, 102 ddir=ddir, force=force, 103 rx=rx, quiet=quiet, 104 legacy=legacy, 105 optimize=optimize, 106 invalidation_mode=invalidation_mode, 107 stripdir=stripdir, 108 prependdir=prependdir, 109 limit_sl_dest=limit_sl_dest, 110 hardlink_dupes=hardlink_dupes), 111 files) 112 success = min(results, default=True) 113 else: 114 for file in files: 115 if not compile_file(file, ddir, force, rx, quiet, 116 legacy, optimize, invalidation_mode, 117 stripdir=stripdir, prependdir=prependdir, 118 limit_sl_dest=limit_sl_dest, 119 hardlink_dupes=hardlink_dupes): 120 success = False 121 return success 122 123def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, 124 legacy=False, optimize=-1, 125 invalidation_mode=None, *, stripdir=None, prependdir=None, 126 limit_sl_dest=None, hardlink_dupes=False): 127 """Byte-compile one file. 128 129 Arguments (only fullname is required): 130 131 fullname: the file to byte-compile 132 ddir: if given, the directory name compiled in to the 133 byte-code file. 134 force: if True, force compilation, even if timestamps are up-to-date 135 quiet: full output with False or 0, errors only with 1, 136 no output with 2 137 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 138 optimize: int or list of optimization levels or -1 for level of 139 the interpreter. Multiple levels leads to multiple compiled 140 files each with one optimization level. 141 invalidation_mode: how the up-to-dateness of the pyc will be checked 142 stripdir: part of path to left-strip from source file path 143 prependdir: path to prepend to beginning of original file path, applied 144 after stripdir 145 limit_sl_dest: ignore symlinks if they are pointing outside of 146 the defined path. 147 hardlink_dupes: hardlink duplicated pyc files 148 """ 149 150 if ddir is not None and (stripdir is not None or prependdir is not None): 151 raise ValueError(("Destination dir (ddir) cannot be used " 152 "in combination with stripdir or prependdir")) 153 154 success = True 155 if quiet < 2 and isinstance(fullname, os.PathLike): 156 fullname = os.fspath(fullname) 157 name = os.path.basename(fullname) 158 159 dfile = None 160 161 if ddir is not None: 162 dfile = os.path.join(ddir, name) 163 164 if stripdir is not None: 165 fullname_parts = fullname.split(os.path.sep) 166 stripdir_parts = stripdir.split(os.path.sep) 167 ddir_parts = list(fullname_parts) 168 169 for spart, opart in zip(stripdir_parts, fullname_parts): 170 if spart == opart: 171 ddir_parts.remove(spart) 172 173 dfile = os.path.join(*ddir_parts) 174 175 if prependdir is not None: 176 if dfile is None: 177 dfile = os.path.join(prependdir, fullname) 178 else: 179 dfile = os.path.join(prependdir, dfile) 180 181 if isinstance(optimize, int): 182 optimize = [optimize] 183 184 # Use set() to remove duplicates. 185 # Use sorted() to create pyc files in a deterministic order. 186 optimize = sorted(set(optimize)) 187 188 if hardlink_dupes and len(optimize) < 2: 189 raise ValueError("Hardlinking of duplicated bytecode makes sense " 190 "only for more than one optimization level") 191 192 if rx is not None: 193 mo = rx.search(fullname) 194 if mo: 195 return success 196 197 if limit_sl_dest is not None and os.path.islink(fullname): 198 if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents: 199 return success 200 201 opt_cfiles = {} 202 203 if os.path.isfile(fullname): 204 for opt_level in optimize: 205 if legacy: 206 opt_cfiles[opt_level] = fullname + 'c' 207 else: 208 if opt_level >= 0: 209 opt = opt_level if opt_level >= 1 else '' 210 cfile = (importlib.util.cache_from_source( 211 fullname, optimization=opt)) 212 opt_cfiles[opt_level] = cfile 213 else: 214 cfile = importlib.util.cache_from_source(fullname) 215 opt_cfiles[opt_level] = cfile 216 217 head, tail = name[:-3], name[-3:] 218 if tail == '.py': 219 if not force: 220 try: 221 mtime = int(os.stat(fullname).st_mtime) 222 expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 223 0, mtime) 224 for cfile in opt_cfiles.values(): 225 with open(cfile, 'rb') as chandle: 226 actual = chandle.read(12) 227 if expect != actual: 228 break 229 else: 230 return success 231 except OSError: 232 pass 233 if not quiet: 234 print('Compiling {!r}...'.format(fullname)) 235 try: 236 for index, opt_level in enumerate(optimize): 237 cfile = opt_cfiles[opt_level] 238 ok = py_compile.compile(fullname, cfile, dfile, True, 239 optimize=opt_level, 240 invalidation_mode=invalidation_mode) 241 if index > 0 and hardlink_dupes: 242 previous_cfile = opt_cfiles[optimize[index - 1]] 243 if filecmp.cmp(cfile, previous_cfile, shallow=False): 244 os.unlink(cfile) 245 os.link(previous_cfile, cfile) 246 except py_compile.PyCompileError as err: 247 success = False 248 if quiet >= 2: 249 return success 250 elif quiet: 251 print('*** Error compiling {!r}...'.format(fullname)) 252 else: 253 print('*** ', end='') 254 # escape non-printable characters in msg 255 msg = err.msg.encode(sys.stdout.encoding, 256 errors='backslashreplace') 257 msg = msg.decode(sys.stdout.encoding) 258 print(msg) 259 except (SyntaxError, UnicodeError, OSError) as e: 260 success = False 261 if quiet >= 2: 262 return success 263 elif quiet: 264 print('*** Error compiling {!r}...'.format(fullname)) 265 else: 266 print('*** ', end='') 267 print(e.__class__.__name__ + ':', e) 268 else: 269 if ok == 0: 270 success = False 271 return success 272 273def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, 274 legacy=False, optimize=-1, 275 invalidation_mode=None): 276 """Byte-compile all module on sys.path. 277 278 Arguments (all optional): 279 280 skip_curdir: if true, skip current directory (default True) 281 maxlevels: max recursion level (default 0) 282 force: as for compile_dir() (default False) 283 quiet: as for compile_dir() (default 0) 284 legacy: as for compile_dir() (default False) 285 optimize: as for compile_dir() (default -1) 286 invalidation_mode: as for compiler_dir() 287 """ 288 success = True 289 for dir in sys.path: 290 if (not dir or dir == os.curdir) and skip_curdir: 291 if quiet < 2: 292 print('Skipping current directory') 293 else: 294 success = success and compile_dir( 295 dir, 296 maxlevels, 297 None, 298 force, 299 quiet=quiet, 300 legacy=legacy, 301 optimize=optimize, 302 invalidation_mode=invalidation_mode, 303 ) 304 return success 305 306 307def main(): 308 """Script main program.""" 309 import argparse 310 311 parser = argparse.ArgumentParser( 312 description='Utilities to support installing Python libraries.') 313 parser.add_argument('-l', action='store_const', const=0, 314 default=None, dest='maxlevels', 315 help="don't recurse into subdirectories") 316 parser.add_argument('-r', type=int, dest='recursion', 317 help=('control the maximum recursion level. ' 318 'if `-l` and `-r` options are specified, ' 319 'then `-r` takes precedence.')) 320 parser.add_argument('-f', action='store_true', dest='force', 321 help='force rebuild even if timestamps are up to date') 322 parser.add_argument('-q', action='count', dest='quiet', default=0, 323 help='output only error messages; -qq will suppress ' 324 'the error messages as well.') 325 parser.add_argument('-b', action='store_true', dest='legacy', 326 help='use legacy (pre-PEP3147) compiled file locations') 327 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, 328 help=('directory to prepend to file paths for use in ' 329 'compile-time tracebacks and in runtime ' 330 'tracebacks in cases where the source file is ' 331 'unavailable')) 332 parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir', 333 default=None, 334 help=('part of path to left-strip from path ' 335 'to source file - for example buildroot. ' 336 '`-d` and `-s` options cannot be ' 337 'specified together.')) 338 parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir', 339 default=None, 340 help=('path to add as prefix to path ' 341 'to source file - for example / to make ' 342 'it absolute when some part is removed ' 343 'by `-s` option. ' 344 '`-d` and `-p` options cannot be ' 345 'specified together.')) 346 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, 347 help=('skip files matching the regular expression; ' 348 'the regexp is searched for in the full path ' 349 'of each file considered for compilation')) 350 parser.add_argument('-i', metavar='FILE', dest='flist', 351 help=('add all the files and directories listed in ' 352 'FILE to the list considered for compilation; ' 353 'if "-", names are read from stdin')) 354 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 355 help=('zero or more file and directory names ' 356 'to compile; if no arguments given, defaults ' 357 'to the equivalent of -l sys.path')) 358 parser.add_argument('-j', '--workers', default=1, 359 type=int, help='Run compileall concurrently') 360 invalidation_modes = [mode.name.lower().replace('_', '-') 361 for mode in py_compile.PycInvalidationMode] 362 parser.add_argument('--invalidation-mode', 363 choices=sorted(invalidation_modes), 364 help=('set .pyc invalidation mode; defaults to ' 365 '"checked-hash" if the SOURCE_DATE_EPOCH ' 366 'environment variable is set, and ' 367 '"timestamp" otherwise.')) 368 parser.add_argument('-o', action='append', type=int, dest='opt_levels', 369 help=('Optimization levels to run compilation with.' 370 'Default is -1 which uses optimization level of' 371 'Python interpreter itself (specified by -O).')) 372 parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', 373 help='Ignore symlinks pointing outsite of the DIR') 374 parser.add_argument('--hardlink-dupes', action='store_true', 375 dest='hardlink_dupes', 376 help='Hardlink duplicated pyc files') 377 378 args = parser.parse_args() 379 compile_dests = args.compile_dest 380 381 if args.rx: 382 import re 383 args.rx = re.compile(args.rx) 384 385 if args.limit_sl_dest == "": 386 args.limit_sl_dest = None 387 388 if args.recursion is not None: 389 maxlevels = args.recursion 390 else: 391 maxlevels = args.maxlevels 392 393 if args.opt_levels is None: 394 args.opt_levels = [-1] 395 396 if len(args.opt_levels) == 1 and args.hardlink_dupes: 397 parser.error(("Hardlinking of duplicated bytecode makes sense " 398 "only for more than one optimization level.")) 399 400 if args.ddir is not None and ( 401 args.stripdir is not None or args.prependdir is not None 402 ): 403 parser.error("-d cannot be used in combination with -s or -p") 404 405 # if flist is provided then load it 406 if args.flist: 407 try: 408 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 409 for line in f: 410 compile_dests.append(line.strip()) 411 except OSError: 412 if args.quiet < 2: 413 print("Error reading file list {}".format(args.flist)) 414 return False 415 416 if args.invalidation_mode: 417 ivl_mode = args.invalidation_mode.replace('-', '_').upper() 418 invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] 419 else: 420 invalidation_mode = None 421 422 success = True 423 try: 424 if compile_dests: 425 for dest in compile_dests: 426 if os.path.isfile(dest): 427 if not compile_file(dest, args.ddir, args.force, args.rx, 428 args.quiet, args.legacy, 429 invalidation_mode=invalidation_mode, 430 stripdir=args.stripdir, 431 prependdir=args.prependdir, 432 optimize=args.opt_levels, 433 limit_sl_dest=args.limit_sl_dest, 434 hardlink_dupes=args.hardlink_dupes): 435 success = False 436 else: 437 if not compile_dir(dest, maxlevels, args.ddir, 438 args.force, args.rx, args.quiet, 439 args.legacy, workers=args.workers, 440 invalidation_mode=invalidation_mode, 441 stripdir=args.stripdir, 442 prependdir=args.prependdir, 443 optimize=args.opt_levels, 444 limit_sl_dest=args.limit_sl_dest, 445 hardlink_dupes=args.hardlink_dupes): 446 success = False 447 return success 448 else: 449 return compile_path(legacy=args.legacy, force=args.force, 450 quiet=args.quiet, 451 invalidation_mode=invalidation_mode) 452 except KeyboardInterrupt: 453 if args.quiet < 2: 454 print("\n[interrupted]") 455 return False 456 return True 457 458 459if __name__ == '__main__': 460 exit_status = int(not main()) 461 sys.exit(exit_status) 462