1"""Module/script to byte-compile all .py files to .pyc files. 2 3When called as a script with arguments, this compiles the directories 4given as arguments recursively; the -l option prevents it from 5recursing into directories. 6 7Without arguments, if compiles all modules on sys.path, without 8recursing into subdirectories. (Even though it should do so for 9packages -- for now, you'll have to deal with packages separately.) 10 11See module py_compile for details of the actual byte-compilation. 12""" 13import os 14import sys 15import importlib.util 16import py_compile 17import struct 18 19from functools import partial 20 21__all__ = ["compile_dir","compile_file","compile_path"] 22 23def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): 24 if quiet < 2 and isinstance(dir, os.PathLike): 25 dir = os.fspath(dir) 26 if not quiet: 27 print('Listing {!r}...'.format(dir)) 28 try: 29 names = os.listdir(dir) 30 except OSError: 31 if quiet < 2: 32 print("Can't list {!r}".format(dir)) 33 names = [] 34 names.sort() 35 for name in names: 36 if name == '__pycache__': 37 continue 38 fullname = os.path.join(dir, name) 39 if ddir is not None: 40 dfile = os.path.join(ddir, name) 41 else: 42 dfile = None 43 if not os.path.isdir(fullname): 44 yield fullname, ddir 45 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and 46 os.path.isdir(fullname) and not os.path.islink(fullname)): 47 yield from _walk_dir(fullname, ddir=dfile, 48 maxlevels=maxlevels - 1, quiet=quiet) 49 50def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, 51 quiet=0, legacy=False, optimize=-1, workers=1, 52 invalidation_mode=None): 53 """Byte-compile all modules in the given directory tree. 54 55 Arguments (only dir is required): 56 57 dir: the directory to byte-compile 58 maxlevels: maximum recursion level (default 10) 59 ddir: the directory that will be prepended to the path to the 60 file as it is compiled into each byte-code file. 61 force: if True, force compilation, even if timestamps are up-to-date 62 quiet: full output with False or 0, errors only with 1, 63 no output with 2 64 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 65 optimize: optimization level or -1 for level of the interpreter 66 workers: maximum number of parallel workers 67 invalidation_mode: how the up-to-dateness of the pyc will be checked 68 """ 69 ProcessPoolExecutor = None 70 if workers < 0: 71 raise ValueError('workers must be greater or equal to 0') 72 if workers != 1: 73 try: 74 # Only import when needed, as low resource platforms may 75 # fail to import it 76 from concurrent.futures import ProcessPoolExecutor 77 except ImportError: 78 workers = 1 79 files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, 80 ddir=ddir) 81 success = True 82 if workers != 1 and ProcessPoolExecutor is not None: 83 # If workers == 0, let ProcessPoolExecutor choose 84 workers = workers or None 85 with ProcessPoolExecutor(max_workers=workers) as executor: 86 results = executor.map( 87 partial(_compile_file_tuple, 88 force=force, rx=rx, quiet=quiet, 89 legacy=legacy, optimize=optimize, 90 invalidation_mode=invalidation_mode, 91 ), 92 files_and_ddirs) 93 success = min(results, default=True) 94 else: 95 for file, dfile in files_and_ddirs: 96 if not compile_file(file, dfile, force, rx, quiet, 97 legacy, optimize, invalidation_mode): 98 success = False 99 return success 100 101def _compile_file_tuple(file_and_dfile, **kwargs): 102 """Needs to be toplevel for ProcessPoolExecutor.""" 103 file, dfile = file_and_dfile 104 return compile_file(file, dfile, **kwargs) 105 106def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, 107 legacy=False, optimize=-1, 108 invalidation_mode=None): 109 """Byte-compile one file. 110 111 Arguments (only fullname is required): 112 113 fullname: the file to byte-compile 114 ddir: if given, the directory name compiled in to the 115 byte-code file. 116 force: if True, force compilation, even if timestamps are up-to-date 117 quiet: full output with False or 0, errors only with 1, 118 no output with 2 119 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 120 optimize: optimization level or -1 for level of the interpreter 121 invalidation_mode: how the up-to-dateness of the pyc will be checked 122 """ 123 success = True 124 if quiet < 2 and isinstance(fullname, os.PathLike): 125 fullname = os.fspath(fullname) 126 name = os.path.basename(fullname) 127 if ddir is not None: 128 dfile = os.path.join(ddir, name) 129 else: 130 dfile = None 131 if rx is not None: 132 mo = rx.search(fullname) 133 if mo: 134 return success 135 if os.path.isfile(fullname): 136 if legacy: 137 cfile = fullname + 'c' 138 else: 139 if optimize >= 0: 140 opt = optimize if optimize >= 1 else '' 141 cfile = importlib.util.cache_from_source( 142 fullname, optimization=opt) 143 else: 144 cfile = importlib.util.cache_from_source(fullname) 145 cache_dir = os.path.dirname(cfile) 146 head, tail = name[:-3], name[-3:] 147 if tail == '.py': 148 if not force: 149 try: 150 mtime = int(os.stat(fullname).st_mtime) 151 expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 152 0, mtime) 153 with open(cfile, 'rb') as chandle: 154 actual = chandle.read(12) 155 if expect == actual: 156 return success 157 except OSError: 158 pass 159 if not quiet: 160 print('Compiling {!r}...'.format(fullname)) 161 try: 162 ok = py_compile.compile(fullname, cfile, dfile, True, 163 optimize=optimize, 164 invalidation_mode=invalidation_mode) 165 except py_compile.PyCompileError as err: 166 success = False 167 if quiet >= 2: 168 return success 169 elif quiet: 170 print('*** Error compiling {!r}...'.format(fullname)) 171 else: 172 print('*** ', end='') 173 # escape non-printable characters in msg 174 msg = err.msg.encode(sys.stdout.encoding, 175 errors='backslashreplace') 176 msg = msg.decode(sys.stdout.encoding) 177 print(msg) 178 except (SyntaxError, UnicodeError, OSError) as e: 179 success = False 180 if quiet >= 2: 181 return success 182 elif quiet: 183 print('*** Error compiling {!r}...'.format(fullname)) 184 else: 185 print('*** ', end='') 186 print(e.__class__.__name__ + ':', e) 187 else: 188 if ok == 0: 189 success = False 190 return success 191 192def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, 193 legacy=False, optimize=-1, 194 invalidation_mode=None): 195 """Byte-compile all module on sys.path. 196 197 Arguments (all optional): 198 199 skip_curdir: if true, skip current directory (default True) 200 maxlevels: max recursion level (default 0) 201 force: as for compile_dir() (default False) 202 quiet: as for compile_dir() (default 0) 203 legacy: as for compile_dir() (default False) 204 optimize: as for compile_dir() (default -1) 205 invalidation_mode: as for compiler_dir() 206 """ 207 success = True 208 for dir in sys.path: 209 if (not dir or dir == os.curdir) and skip_curdir: 210 if quiet < 2: 211 print('Skipping current directory') 212 else: 213 success = success and compile_dir( 214 dir, 215 maxlevels, 216 None, 217 force, 218 quiet=quiet, 219 legacy=legacy, 220 optimize=optimize, 221 invalidation_mode=invalidation_mode, 222 ) 223 return success 224 225 226def main(): 227 """Script main program.""" 228 import argparse 229 230 parser = argparse.ArgumentParser( 231 description='Utilities to support installing Python libraries.') 232 parser.add_argument('-l', action='store_const', const=0, 233 default=10, dest='maxlevels', 234 help="don't recurse into subdirectories") 235 parser.add_argument('-r', type=int, dest='recursion', 236 help=('control the maximum recursion level. ' 237 'if `-l` and `-r` options are specified, ' 238 'then `-r` takes precedence.')) 239 parser.add_argument('-f', action='store_true', dest='force', 240 help='force rebuild even if timestamps are up to date') 241 parser.add_argument('-q', action='count', dest='quiet', default=0, 242 help='output only error messages; -qq will suppress ' 243 'the error messages as well.') 244 parser.add_argument('-b', action='store_true', dest='legacy', 245 help='use legacy (pre-PEP3147) compiled file locations') 246 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, 247 help=('directory to prepend to file paths for use in ' 248 'compile-time tracebacks and in runtime ' 249 'tracebacks in cases where the source file is ' 250 'unavailable')) 251 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, 252 help=('skip files matching the regular expression; ' 253 'the regexp is searched for in the full path ' 254 'of each file considered for compilation')) 255 parser.add_argument('-i', metavar='FILE', dest='flist', 256 help=('add all the files and directories listed in ' 257 'FILE to the list considered for compilation; ' 258 'if "-", names are read from stdin')) 259 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 260 help=('zero or more file and directory names ' 261 'to compile; if no arguments given, defaults ' 262 'to the equivalent of -l sys.path')) 263 parser.add_argument('-j', '--workers', default=1, 264 type=int, help='Run compileall concurrently') 265 invalidation_modes = [mode.name.lower().replace('_', '-') 266 for mode in py_compile.PycInvalidationMode] 267 parser.add_argument('--invalidation-mode', 268 choices=sorted(invalidation_modes), 269 help=('set .pyc invalidation mode; defaults to ' 270 '"checked-hash" if the SOURCE_DATE_EPOCH ' 271 'environment variable is set, and ' 272 '"timestamp" otherwise.')) 273 274 args = parser.parse_args() 275 compile_dests = args.compile_dest 276 277 if args.rx: 278 import re 279 args.rx = re.compile(args.rx) 280 281 282 if args.recursion is not None: 283 maxlevels = args.recursion 284 else: 285 maxlevels = args.maxlevels 286 287 # if flist is provided then load it 288 if args.flist: 289 try: 290 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 291 for line in f: 292 compile_dests.append(line.strip()) 293 except OSError: 294 if args.quiet < 2: 295 print("Error reading file list {}".format(args.flist)) 296 return False 297 298 if args.invalidation_mode: 299 ivl_mode = args.invalidation_mode.replace('-', '_').upper() 300 invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] 301 else: 302 invalidation_mode = None 303 304 success = True 305 try: 306 if compile_dests: 307 for dest in compile_dests: 308 if os.path.isfile(dest): 309 if not compile_file(dest, args.ddir, args.force, args.rx, 310 args.quiet, args.legacy, 311 invalidation_mode=invalidation_mode): 312 success = False 313 else: 314 if not compile_dir(dest, maxlevels, args.ddir, 315 args.force, args.rx, args.quiet, 316 args.legacy, workers=args.workers, 317 invalidation_mode=invalidation_mode): 318 success = False 319 return success 320 else: 321 return compile_path(legacy=args.legacy, force=args.force, 322 quiet=args.quiet, 323 invalidation_mode=invalidation_mode) 324 except KeyboardInterrupt: 325 if args.quiet < 2: 326 print("\n[interrupted]") 327 return False 328 return True 329 330 331if __name__ == '__main__': 332 exit_status = int(not main()) 333 sys.exit(exit_status) 334