Lines Matching full:token

27         token = lexer.nextToken()
28         assert token.type == self.lexerModule.IDENTIFIER
29         assert token.start == 0, token.start
30         assert token.stop == 5, token.stop
31         assert token.text == 'foobar', token.text
33         token = lexer.nextToken()
34         assert token.type == self.lexerModule.WS
35         assert token.start == 6, token.start
36         assert token.stop == 6, token.stop
37         assert token.text == ' ', token.text
39         token = lexer.nextToken()
40         assert token.type == self.lexerModule.IDENTIFIER
41         assert token.start == 7, token.start
42         assert token.stop == 11, token.stop
43         assert token.text == '_Ab98', token.text
45         token = lexer.nextToken()
46         assert token.type == self.lexerModule.WS
47         assert token.start == 12, token.start
48         assert token.stop == 14, token.stop
49         assert token.text == ' \n ', token.text
51         token = lexer.nextToken()
52         assert token.type == self.lexerModule.IDENTIFIER
53         assert token.start == 15, token.start
54         assert token.stop == 20, token.stop
55         assert token.text == 'A12sdf', token.text
57         token = lexer.nextToken()
58         assert token.type == self.lexerModule.EOF
67             token = lexer.nextToken()
68             raise AssertionError, token