"""ANTLR3 runtime package""" # begin[licence] # # [The "BSD licence"] # Copyright (c) 2005-2012 Terence Parr # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. The name of the author may not be used to endorse or promote products # derived from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # end[licence] from .constants import DEFAULT_CHANNEL, EOF, INVALID_TOKEN_TYPE ############################################################################ # # basic token interface # ############################################################################ class Token(object): """@brief Abstract token baseclass.""" TOKEN_NAMES_MAP = None @classmethod def registerTokenNamesMap(cls, tokenNamesMap): """@brief Store a mapping from token type to token name. This enables token.typeName to give something more meaningful than, e.g., '6'. """ cls.TOKEN_NAMES_MAP = tokenNamesMap cls.TOKEN_NAMES_MAP[EOF] = "EOF" def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None, index=-1, line=0, charPositionInLine=-1, input=None): # We use -1 for index and charPositionInLine as an invalid index self._type = type self._channel = channel self._text = text self._index = index self._line = 0 self._charPositionInLine = charPositionInLine self.input = input # To override a property, you'll need to override both the getter and setter. @property def text(self): return self._text @text.setter def text(self, value): self._text = value @property def type(self): return self._type @type.setter def type(self, value): self._type = value # For compatibility def getType(self): return self._type @property def typeName(self): if self.TOKEN_NAMES_MAP: return self.TOKEN_NAMES_MAP.get(self._type, "INVALID_TOKEN_TYPE") else: return str(self._type) @property def line(self): """Lines are numbered 1..n.""" return self._line @line.setter def line(self, value): self._line = value @property def charPositionInLine(self): """Columns are numbered 0..n-1.""" return self._charPositionInLine @charPositionInLine.setter def charPositionInLine(self, pos): self._charPositionInLine = pos @property def channel(self): return self._channel @channel.setter def channel(self, value): self._channel = value @property def index(self): """ An index from 0..n-1 of the token object in the input stream. This must be valid in order to use the ANTLRWorks debugger. """ return self._index @index.setter def index(self, value): self._index = value def getInputStream(self): """@brief From what character stream was this token created. You don't have to implement but it's nice to know where a Token comes from if you have include files etc... on the input.""" raise NotImplementedError def setInputStream(self, input): """@brief From what character stream was this token created. You don't have to implement but it's nice to know where a Token comes from if you have include files etc... on the input.""" raise NotImplementedError ############################################################################ # # token implementations # # Token # +- CommonToken # \- ClassicToken # ############################################################################ class CommonToken(Token): """@brief Basic token implementation. This implementation does not copy the text from the input stream upon creation, but keeps start/stop pointers into the stream to avoid unnecessary copy operations. """ def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None, input=None, start=None, stop=None, oldToken=None): if oldToken: super().__init__(oldToken.type, oldToken.channel, oldToken.text, oldToken.index, oldToken.line, oldToken.charPositionInLine, oldToken.input) if isinstance(oldToken, CommonToken): self.start = oldToken.start self.stop = oldToken.stop else: self.start = start self.stop = stop else: super().__init__(type=type, channel=channel, input=input) # We need to be able to change the text once in a while. If # this is non-null, then getText should return this. Note that # start/stop are not affected by changing this. self._text = text # The char position into the input buffer where this token starts self.start = start # The char position into the input buffer where this token stops # This is the index of the last char, *not* the index after it! self.stop = stop @property def text(self): # Could be the empty string, and we want to return that. if self._text is not None: return self._text if not self.input: return None if self.start < self.input.size() and self.stop < self.input.size(): return self.input.substring(self.start, self.stop) return '' @text.setter def text(self, value): """ Override the text for this token. getText() will return this text rather than pulling from the buffer. Note that this does not mean that start/stop indexes are not valid. It means that that input was converted to a new string in the token object. """ self._text = value def getInputStream(self): return self.input def setInputStream(self, input): self.input = input def __str__(self): if self.type == EOF: return "" channelStr = "" if self.channel > 0: channelStr = ",channel=" + str(self.channel) txt = self.text if txt: # Put 2 backslashes in front of each character txt = txt.replace("\n", r"\\n") txt = txt.replace("\r", r"\\r") txt = txt.replace("\t", r"\\t") else: txt = "" return ("[@{0.index},{0.start}:{0.stop}={txt!r}," "<{0.typeName}>{channelStr}," "{0.line}:{0.charPositionInLine}]" .format(self, txt=txt, channelStr=channelStr)) class ClassicToken(Token): """@brief Alternative token implementation. A Token object like we'd use in ANTLR 2.x; has an actual string created and associated with this object. These objects are needed for imaginary tree nodes that have payload objects. We need to create a Token object that has a string; the tree node will point at this token. CommonToken has indexes into a char stream and hence cannot be used to introduce new strings. """ def __init__(self, type=None, text=None, channel=DEFAULT_CHANNEL, oldToken=None): if oldToken: super().__init__(type=oldToken.type, channel=oldToken.channel, text=oldToken.text, line=oldToken.line, charPositionInLine=oldToken.charPositionInLine) else: super().__init__(type=type, channel=channel, text=text, index=None, line=None, charPositionInLine=None) def getInputStream(self): return None def setInputStream(self, input): pass def toString(self): channelStr = "" if self.channel > 0: channelStr = ",channel=" + str(self.channel) txt = self.text if not txt: txt = "" return ("[@{0.index!r},{txt!r},<{0.type!r}>{channelStr}," "{0.line!r}:{0.charPositionInLine!r}]" .format(self, txt=txt, channelStr=channelStr)) __str__ = toString __repr__ = toString INVALID_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE) # In an action, a lexer rule can set token to this SKIP_TOKEN and ANTLR # will avoid creating a token for this symbol and try to fetch another. SKIP_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)