• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
2# Licensed to PSF under a Contributor Agreement.
3
4# Modifications:
5# Copyright 2006 Google, Inc. All Rights Reserved.
6# Licensed to PSF under a Contributor Agreement.
7
8"""Parser driver.
9
10This provides a high-level interface to parse a file into a syntax tree.
11
12"""
13
14__author__ = "Guido van Rossum <guido@python.org>"
15
16__all__ = ["Driver", "load_grammar"]
17
18# Python imports
19import codecs
20import os
21import logging
22import StringIO
23import sys
24
25# Pgen imports
26from . import grammar, parse, token, tokenize, pgen
27
28
29class Driver(object):
30
31    def __init__(self, grammar, convert=None, logger=None):
32        self.grammar = grammar
33        if logger is None:
34            logger = logging.getLogger()
35        self.logger = logger
36        self.convert = convert
37
38    def parse_tokens(self, tokens, debug=False):
39        """Parse a series of tokens and return the syntax tree."""
40        # XXX Move the prefix computation into a wrapper around tokenize.
41        p = parse.Parser(self.grammar, self.convert)
42        p.setup()
43        lineno = 1
44        column = 0
45        type = value = start = end = line_text = None
46        prefix = u""
47        for quintuple in tokens:
48            type, value, start, end, line_text = quintuple
49            if start != (lineno, column):
50                assert (lineno, column) <= start, ((lineno, column), start)
51                s_lineno, s_column = start
52                if lineno < s_lineno:
53                    prefix += "\n" * (s_lineno - lineno)
54                    lineno = s_lineno
55                    column = 0
56                if column < s_column:
57                    prefix += line_text[column:s_column]
58                    column = s_column
59            if type in (tokenize.COMMENT, tokenize.NL):
60                prefix += value
61                lineno, column = end
62                if value.endswith("\n"):
63                    lineno += 1
64                    column = 0
65                continue
66            if type == token.OP:
67                type = grammar.opmap[value]
68            if debug:
69                self.logger.debug("%s %r (prefix=%r)",
70                                  token.tok_name[type], value, prefix)
71            if p.addtoken(type, value, (prefix, start)):
72                if debug:
73                    self.logger.debug("Stop.")
74                break
75            prefix = ""
76            lineno, column = end
77            if value.endswith("\n"):
78                lineno += 1
79                column = 0
80        else:
81            # We never broke out -- EOF is too soon (how can this happen???)
82            raise parse.ParseError("incomplete input",
83                                   type, value, (prefix, start))
84        return p.rootnode
85
86    def parse_stream_raw(self, stream, debug=False):
87        """Parse a stream and return the syntax tree."""
88        tokens = tokenize.generate_tokens(stream.readline)
89        return self.parse_tokens(tokens, debug)
90
91    def parse_stream(self, stream, debug=False):
92        """Parse a stream and return the syntax tree."""
93        return self.parse_stream_raw(stream, debug)
94
95    def parse_file(self, filename, encoding=None, debug=False):
96        """Parse a file and return the syntax tree."""
97        stream = codecs.open(filename, "r", encoding)
98        try:
99            return self.parse_stream(stream, debug)
100        finally:
101            stream.close()
102
103    def parse_string(self, text, debug=False):
104        """Parse a string and return the syntax tree."""
105        tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline)
106        return self.parse_tokens(tokens, debug)
107
108
109def _generate_pickle_name(gt):
110    head, tail = os.path.splitext(gt)
111    if tail == ".txt":
112        tail = ""
113    return head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
114
115
116def load_grammar(gt="Grammar.txt", gp=None,
117                 save=True, force=False, logger=None):
118    """Load the grammar (maybe from a pickle)."""
119    if logger is None:
120        logger = logging.getLogger()
121    gp = _generate_pickle_name(gt) if gp is None else gp
122    if force or not _newer(gp, gt):
123        logger.info("Generating grammar tables from %s", gt)
124        g = pgen.generate_grammar(gt)
125        if save:
126            logger.info("Writing grammar tables to %s", gp)
127            try:
128                g.dump(gp)
129            except IOError as e:
130                logger.info("Writing failed: %s", e)
131    else:
132        g = grammar.Grammar()
133        g.load(gp)
134    return g
135
136
137def _newer(a, b):
138    """Inquire whether file a was written since file b."""
139    if not os.path.exists(a):
140        return False
141    if not os.path.exists(b):
142        return True
143    return os.path.getmtime(a) >= os.path.getmtime(b)
144
145
146def main(*args):
147    """Main program, when run as a script: produce grammar pickle files.
148
149    Calls load_grammar for each argument, a path to a grammar text file.
150    """
151    if not args:
152        args = sys.argv[1:]
153    logging.basicConfig(level=logging.INFO, stream=sys.stdout,
154                        format='%(message)s')
155    for gt in args:
156        load_grammar(gt, save=True, force=True)
157    return True
158
159if __name__ == "__main__":
160    sys.exit(int(not main()))
161