• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Generate Lib/keyword.py from the Grammar and Tokens files using pgen"""
2
3import argparse
4
5from .build import build_parser, generate_token_definitions
6from .c_generator import CParserGenerator
7
8TEMPLATE = r'''
9"""Keywords (from "Grammar/python.gram")
10
11This file is automatically generated; please don't muck it up!
12
13To update the symbols in this file, 'cd' to the top directory of
14the python source tree and run:
15
16    PYTHONPATH=Tools/peg_generator python3 -m pegen.keywordgen \
17        Grammar/Grammar \
18        Grammar/Tokens \
19        Lib/keyword.py
20
21Alternatively, you can run 'make regen-keyword'.
22"""
23
24__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
25
26kwlist = [
27{keywords}
28]
29
30softkwlist = [
31{soft_keywords}
32]
33
34iskeyword = frozenset(kwlist).__contains__
35issoftkeyword = frozenset(softkwlist).__contains__
36'''.lstrip()
37
38EXTRA_KEYWORDS = ["async", "await"]
39
40
41def main():
42    parser = argparse.ArgumentParser(
43        description="Generate the Lib/keywords.py file from the grammar."
44    )
45    parser.add_argument(
46        "grammar", type=str, help="The file with the grammar definition in PEG format"
47    )
48    parser.add_argument(
49        "tokens_file",
50        type=argparse.FileType("r"),
51        help="The file with the token definitions"
52    )
53    parser.add_argument(
54        "keyword_file",
55        type=argparse.FileType("w"),
56        help="The path to write the keyword definitions",
57    )
58    args = parser.parse_args()
59
60    grammar, _, _ = build_parser(args.grammar)
61    with args.tokens_file as tok_file:
62        all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file)
63    gen: ParserGenerator = CParserGenerator(
64        grammar, all_tokens, exact_tok, non_exact_tok, file=None
65    )
66    gen.collect_todo()
67
68    with args.keyword_file as thefile:
69        all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS)
70        all_soft_keywords = sorted(gen.callmakervisitor.soft_keywords)
71
72        keywords = "" if not all_keywords else "    " + ",\n    ".join(map(repr, all_keywords))
73        soft_keywords = "" if not all_soft_keywords else "    " + ",\n    ".join(map(repr, all_soft_keywords))
74        thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords))
75
76
77if __name__ == "__main__":
78    main()
79