1"""Module/script to byte-compile all .py files to .pyc files. 2 3When called as a script with arguments, this compiles the directories 4given as arguments recursively; the -l option prevents it from 5recursing into directories. 6 7Without arguments, if compiles all modules on sys.path, without 8recursing into subdirectories. (Even though it should do so for 9packages -- for now, you'll have to deal with packages separately.) 10 11See module py_compile for details of the actual byte-compilation. 12""" 13import os 14import sys 15import importlib.util 16import py_compile 17import struct 18 19from functools import partial 20 21__all__ = ["compile_dir","compile_file","compile_path"] 22 23def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): 24 if quiet < 2 and isinstance(dir, os.PathLike): 25 dir = os.fspath(dir) 26 if not quiet: 27 print('Listing {!r}...'.format(dir)) 28 try: 29 names = os.listdir(dir) 30 except OSError: 31 if quiet < 2: 32 print("Can't list {!r}".format(dir)) 33 names = [] 34 names.sort() 35 for name in names: 36 if name == '__pycache__': 37 continue 38 fullname = os.path.join(dir, name) 39 if ddir is not None: 40 dfile = os.path.join(ddir, name) 41 else: 42 dfile = None 43 if not os.path.isdir(fullname): 44 yield fullname 45 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and 46 os.path.isdir(fullname) and not os.path.islink(fullname)): 47 yield from _walk_dir(fullname, ddir=dfile, 48 maxlevels=maxlevels - 1, quiet=quiet) 49 50def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, 51 quiet=0, legacy=False, optimize=-1, workers=1, 52 invalidation_mode=None): 53 """Byte-compile all modules in the given directory tree. 54 55 Arguments (only dir is required): 56 57 dir: the directory to byte-compile 58 maxlevels: maximum recursion level (default 10) 59 ddir: the directory that will be prepended to the path to the 60 file as it is compiled into each byte-code file. 61 force: if True, force compilation, even if timestamps are up-to-date 62 quiet: full output with False or 0, errors only with 1, 63 no output with 2 64 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 65 optimize: optimization level or -1 for level of the interpreter 66 workers: maximum number of parallel workers 67 invalidation_mode: how the up-to-dateness of the pyc will be checked 68 """ 69 ProcessPoolExecutor = None 70 if workers < 0: 71 raise ValueError('workers must be greater or equal to 0') 72 if workers != 1: 73 try: 74 # Only import when needed, as low resource platforms may 75 # fail to import it 76 from concurrent.futures import ProcessPoolExecutor 77 except ImportError: 78 workers = 1 79 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, 80 ddir=ddir) 81 success = True 82 if workers != 1 and ProcessPoolExecutor is not None: 83 # If workers == 0, let ProcessPoolExecutor choose 84 workers = workers or None 85 with ProcessPoolExecutor(max_workers=workers) as executor: 86 results = executor.map(partial(compile_file, 87 ddir=ddir, force=force, 88 rx=rx, quiet=quiet, 89 legacy=legacy, 90 optimize=optimize, 91 invalidation_mode=invalidation_mode), 92 files) 93 success = min(results, default=True) 94 else: 95 for file in files: 96 if not compile_file(file, ddir, force, rx, quiet, 97 legacy, optimize, invalidation_mode): 98 success = False 99 return success 100 101def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, 102 legacy=False, optimize=-1, 103 invalidation_mode=None): 104 """Byte-compile one file. 105 106 Arguments (only fullname is required): 107 108 fullname: the file to byte-compile 109 ddir: if given, the directory name compiled in to the 110 byte-code file. 111 force: if True, force compilation, even if timestamps are up-to-date 112 quiet: full output with False or 0, errors only with 1, 113 no output with 2 114 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 115 optimize: optimization level or -1 for level of the interpreter 116 invalidation_mode: how the up-to-dateness of the pyc will be checked 117 """ 118 success = True 119 if quiet < 2 and isinstance(fullname, os.PathLike): 120 fullname = os.fspath(fullname) 121 name = os.path.basename(fullname) 122 if ddir is not None: 123 dfile = os.path.join(ddir, name) 124 else: 125 dfile = None 126 if rx is not None: 127 mo = rx.search(fullname) 128 if mo: 129 return success 130 if os.path.isfile(fullname): 131 if legacy: 132 cfile = fullname + 'c' 133 else: 134 if optimize >= 0: 135 opt = optimize if optimize >= 1 else '' 136 cfile = importlib.util.cache_from_source( 137 fullname, optimization=opt) 138 else: 139 cfile = importlib.util.cache_from_source(fullname) 140 cache_dir = os.path.dirname(cfile) 141 head, tail = name[:-3], name[-3:] 142 if tail == '.py': 143 if not force: 144 try: 145 mtime = int(os.stat(fullname).st_mtime) 146 expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 147 0, mtime) 148 with open(cfile, 'rb') as chandle: 149 actual = chandle.read(12) 150 if expect == actual: 151 return success 152 except OSError: 153 pass 154 if not quiet: 155 print('Compiling {!r}...'.format(fullname)) 156 try: 157 ok = py_compile.compile(fullname, cfile, dfile, True, 158 optimize=optimize, 159 invalidation_mode=invalidation_mode) 160 except py_compile.PyCompileError as err: 161 success = False 162 if quiet >= 2: 163 return success 164 elif quiet: 165 print('*** Error compiling {!r}...'.format(fullname)) 166 else: 167 print('*** ', end='') 168 # escape non-printable characters in msg 169 msg = err.msg.encode(sys.stdout.encoding, 170 errors='backslashreplace') 171 msg = msg.decode(sys.stdout.encoding) 172 print(msg) 173 except (SyntaxError, UnicodeError, OSError) as e: 174 success = False 175 if quiet >= 2: 176 return success 177 elif quiet: 178 print('*** Error compiling {!r}...'.format(fullname)) 179 else: 180 print('*** ', end='') 181 print(e.__class__.__name__ + ':', e) 182 else: 183 if ok == 0: 184 success = False 185 return success 186 187def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, 188 legacy=False, optimize=-1, 189 invalidation_mode=None): 190 """Byte-compile all module on sys.path. 191 192 Arguments (all optional): 193 194 skip_curdir: if true, skip current directory (default True) 195 maxlevels: max recursion level (default 0) 196 force: as for compile_dir() (default False) 197 quiet: as for compile_dir() (default 0) 198 legacy: as for compile_dir() (default False) 199 optimize: as for compile_dir() (default -1) 200 invalidation_mode: as for compiler_dir() 201 """ 202 success = True 203 for dir in sys.path: 204 if (not dir or dir == os.curdir) and skip_curdir: 205 if quiet < 2: 206 print('Skipping current directory') 207 else: 208 success = success and compile_dir( 209 dir, 210 maxlevels, 211 None, 212 force, 213 quiet=quiet, 214 legacy=legacy, 215 optimize=optimize, 216 invalidation_mode=invalidation_mode, 217 ) 218 return success 219 220 221def main(): 222 """Script main program.""" 223 import argparse 224 225 parser = argparse.ArgumentParser( 226 description='Utilities to support installing Python libraries.') 227 parser.add_argument('-l', action='store_const', const=0, 228 default=10, dest='maxlevels', 229 help="don't recurse into subdirectories") 230 parser.add_argument('-r', type=int, dest='recursion', 231 help=('control the maximum recursion level. ' 232 'if `-l` and `-r` options are specified, ' 233 'then `-r` takes precedence.')) 234 parser.add_argument('-f', action='store_true', dest='force', 235 help='force rebuild even if timestamps are up to date') 236 parser.add_argument('-q', action='count', dest='quiet', default=0, 237 help='output only error messages; -qq will suppress ' 238 'the error messages as well.') 239 parser.add_argument('-b', action='store_true', dest='legacy', 240 help='use legacy (pre-PEP3147) compiled file locations') 241 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, 242 help=('directory to prepend to file paths for use in ' 243 'compile-time tracebacks and in runtime ' 244 'tracebacks in cases where the source file is ' 245 'unavailable')) 246 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, 247 help=('skip files matching the regular expression; ' 248 'the regexp is searched for in the full path ' 249 'of each file considered for compilation')) 250 parser.add_argument('-i', metavar='FILE', dest='flist', 251 help=('add all the files and directories listed in ' 252 'FILE to the list considered for compilation; ' 253 'if "-", names are read from stdin')) 254 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 255 help=('zero or more file and directory names ' 256 'to compile; if no arguments given, defaults ' 257 'to the equivalent of -l sys.path')) 258 parser.add_argument('-j', '--workers', default=1, 259 type=int, help='Run compileall concurrently') 260 invalidation_modes = [mode.name.lower().replace('_', '-') 261 for mode in py_compile.PycInvalidationMode] 262 parser.add_argument('--invalidation-mode', 263 choices=sorted(invalidation_modes), 264 help=('set .pyc invalidation mode; defaults to ' 265 '"checked-hash" if the SOURCE_DATE_EPOCH ' 266 'environment variable is set, and ' 267 '"timestamp" otherwise.')) 268 269 args = parser.parse_args() 270 compile_dests = args.compile_dest 271 272 if args.rx: 273 import re 274 args.rx = re.compile(args.rx) 275 276 277 if args.recursion is not None: 278 maxlevels = args.recursion 279 else: 280 maxlevels = args.maxlevels 281 282 # if flist is provided then load it 283 if args.flist: 284 try: 285 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 286 for line in f: 287 compile_dests.append(line.strip()) 288 except OSError: 289 if args.quiet < 2: 290 print("Error reading file list {}".format(args.flist)) 291 return False 292 293 if args.invalidation_mode: 294 ivl_mode = args.invalidation_mode.replace('-', '_').upper() 295 invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] 296 else: 297 invalidation_mode = None 298 299 success = True 300 try: 301 if compile_dests: 302 for dest in compile_dests: 303 if os.path.isfile(dest): 304 if not compile_file(dest, args.ddir, args.force, args.rx, 305 args.quiet, args.legacy, 306 invalidation_mode=invalidation_mode): 307 success = False 308 else: 309 if not compile_dir(dest, maxlevels, args.ddir, 310 args.force, args.rx, args.quiet, 311 args.legacy, workers=args.workers, 312 invalidation_mode=invalidation_mode): 313 success = False 314 return success 315 else: 316 return compile_path(legacy=args.legacy, force=args.force, 317 quiet=args.quiet, 318 invalidation_mode=invalidation_mode) 319 except KeyboardInterrupt: 320 if args.quiet < 2: 321 print("\n[interrupted]") 322 return False 323 return True 324 325 326if __name__ == '__main__': 327 exit_status = int(not main()) 328 sys.exit(exit_status) 329