1#!/usr/bin/env python3.8 2 3import argparse 4import ast 5import os 6import sys 7import time 8import traceback 9import tokenize 10from glob import glob, escape 11from pathlib import PurePath 12 13from typing import List, Optional, Any, Tuple 14 15sys.path.insert(0, os.getcwd()) 16from pegen.ast_dump import ast_dump 17from pegen.testutil import print_memstats 18 19SUCCESS = "\033[92m" 20FAIL = "\033[91m" 21ENDC = "\033[0m" 22 23COMPILE = 2 24PARSE = 1 25NOTREE = 0 26 27argparser = argparse.ArgumentParser( 28 prog="test_parse_directory", 29 description="Helper program to test directories or files for pegen", 30) 31argparser.add_argument("-d", "--directory", help="Directory path containing files to test") 32argparser.add_argument( 33 "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude" 34) 35argparser.add_argument( 36 "-s", "--short", action="store_true", help="Only show errors, in a more Emacs-friendly format" 37) 38argparser.add_argument( 39 "-v", "--verbose", action="store_true", help="Display detailed errors for failures" 40) 41 42 43def report_status( 44 succeeded: bool, 45 file: str, 46 verbose: bool, 47 error: Optional[Exception] = None, 48 short: bool = False, 49) -> None: 50 if short and succeeded: 51 return 52 53 if succeeded is True: 54 status = "OK" 55 COLOR = SUCCESS 56 else: 57 status = "Fail" 58 COLOR = FAIL 59 60 if short: 61 lineno = 0 62 offset = 0 63 if isinstance(error, SyntaxError): 64 lineno = error.lineno or 1 65 offset = error.offset or 1 66 message = error.args[0] 67 else: 68 message = f"{error.__class__.__name__}: {error}" 69 print(f"{file}:{lineno}:{offset}: {message}") 70 else: 71 print(f"{COLOR}{file:60} {status}{ENDC}") 72 73 if error and verbose: 74 print(f" {str(error.__class__.__name__)}: {error}") 75 76 77def parse_file(source: str, file: str) -> Tuple[Any, float]: 78 t0 = time.time() 79 result = ast.parse(source, filename=file) 80 t1 = time.time() 81 return result, t1 - t0 82 83 84def generate_time_stats(files, total_seconds) -> None: 85 total_files = len(files) 86 total_bytes = 0 87 total_lines = 0 88 for file in files: 89 # Count lines and bytes separately 90 with open(file, "rb") as f: 91 total_lines += sum(1 for _ in f) 92 total_bytes += f.tell() 93 94 print( 95 f"Checked {total_files:,} files, {total_lines:,} lines,", 96 f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", 97 ) 98 if total_seconds > 0: 99 print( 100 f"That's {total_lines / total_seconds :,.0f} lines/sec,", 101 f"or {total_bytes / total_seconds :,.0f} bytes/sec.", 102 ) 103 104 105def parse_directory(directory: str, verbose: bool, excluded_files: List[str], short: bool) -> int: 106 # For a given directory, traverse files and attempt to parse each one 107 # - Output success/failure for each file 108 errors = 0 109 files = [] 110 total_seconds = 0 111 112 for file in sorted(glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)): 113 # Only attempt to parse Python files and files that are not excluded 114 if any(PurePath(file).match(pattern) for pattern in excluded_files): 115 continue 116 117 with tokenize.open(file) as f: 118 source = f.read() 119 120 try: 121 result, dt = parse_file(source, file) 122 total_seconds += dt 123 report_status(succeeded=True, file=file, verbose=verbose, short=short) 124 except SyntaxError as error: 125 report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short) 126 errors += 1 127 files.append(file) 128 129 generate_time_stats(files, total_seconds) 130 if short: 131 print_memstats() 132 133 if errors: 134 print(f"Encountered {errors} failures.", file=sys.stderr) 135 return 1 136 137 return 0 138 139 140def main() -> None: 141 args = argparser.parse_args() 142 directory = args.directory 143 verbose = args.verbose 144 excluded_files = args.exclude 145 short = args.short 146 sys.exit(parse_directory(directory, verbose, excluded_files, short)) 147 148 149if __name__ == "__main__": 150 main() 151