1 import antlr3 2 import testbase 3 import unittest 4 5 class t010lexer(testbase.ANTLRTest): 6 def setUp(self): 7 self.compileGrammar() 8 9 10 def lexerClass(self, base): 11 class TLexer(base): 12 def emitErrorMessage(self, msg): 13 # report errors to /dev/null 14 pass 15 16 def reportError(self, re): 17 # no error recovery yet, just crash! 18 raise re 19 20 return TLexer 21 22 23 def testValid(self): 24 stream = antlr3.StringStream('foobar _Ab98 \n A12sdf') 25 lexer = self.getLexer(stream) 26 27 token = lexer.nextToken() 28 assert token.type == self.lexerModule.IDENTIFIER 29 assert token.start == 0, token.start 30 assert token.stop == 5, token.stop 31 assert token.text == 'foobar', token.text 32 33 token = lexer.nextToken() 34 assert token.type == self.lexerModule.WS 35 assert token.start == 6, token.start 36 assert token.stop == 6, token.stop 37 assert token.text == ' ', token.text 38 39 token = lexer.nextToken() 40 assert token.type == self.lexerModule.IDENTIFIER 41 assert token.start == 7, token.start 42 assert token.stop == 11, token.stop 43 assert token.text == '_Ab98', token.text 44 45 token = lexer.nextToken() 46 assert token.type == self.lexerModule.WS 47 assert token.start == 12, token.start 48 assert token.stop == 14, token.stop 49 assert token.text == ' \n ', token.text 50 51 token = lexer.nextToken() 52 assert token.type == self.lexerModule.IDENTIFIER 53 assert token.start == 15, token.start 54 assert token.stop == 20, token.stop 55 assert token.text == 'A12sdf', token.text 56 57 token = lexer.nextToken() 58 assert token.type == self.lexerModule.EOF 59 60 61 def testMalformedInput(self): 62 stream = antlr3.StringStream('a-b') 63 lexer = self.getLexer(stream) 64 65 lexer.nextToken() 66 try: 67 token = lexer.nextToken() 68 raise AssertionError, token 69 70 except antlr3.NoViableAltException, exc: 71 assert exc.unexpectedType == '-', repr(exc.unexpectedType) 72 assert exc.charPositionInLine == 1, repr(exc.charPositionInLine) 73 assert exc.line == 1, repr(exc.line) 74 75 76 77 if __name__ == '__main__': 78 unittest.main() 79