• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2
3import os
4import unittest
5from StringIO import StringIO
6import antlr3
7
8
9class TestStringStream(unittest.TestCase):
10    """Test case for the StringStream class."""
11
12    def testSize(self):
13        """StringStream.size()"""
14
15        stream = antlr3.StringStream('foo')
16
17        self.failUnlessEqual(stream.size(), 3)
18
19
20    def testIndex(self):
21        """StringStream.index()"""
22
23        stream = antlr3.StringStream('foo')
24
25        self.failUnlessEqual(stream.index(), 0)
26
27
28    def testConsume(self):
29        """StringStream.consume()"""
30
31        stream = antlr3.StringStream('foo\nbar')
32
33        stream.consume() # f
34        self.failUnlessEqual(stream.index(), 1)
35        self.failUnlessEqual(stream.charPositionInLine, 1)
36        self.failUnlessEqual(stream.line, 1)
37
38        stream.consume() # o
39        self.failUnlessEqual(stream.index(), 2)
40        self.failUnlessEqual(stream.charPositionInLine, 2)
41        self.failUnlessEqual(stream.line, 1)
42
43        stream.consume() # o
44        self.failUnlessEqual(stream.index(), 3)
45        self.failUnlessEqual(stream.charPositionInLine, 3)
46        self.failUnlessEqual(stream.line, 1)
47
48        stream.consume() # \n
49        self.failUnlessEqual(stream.index(), 4)
50        self.failUnlessEqual(stream.charPositionInLine, 0)
51        self.failUnlessEqual(stream.line, 2)
52
53        stream.consume() # b
54        self.failUnlessEqual(stream.index(), 5)
55        self.failUnlessEqual(stream.charPositionInLine, 1)
56        self.failUnlessEqual(stream.line, 2)
57
58        stream.consume() # a
59        self.failUnlessEqual(stream.index(), 6)
60        self.failUnlessEqual(stream.charPositionInLine, 2)
61        self.failUnlessEqual(stream.line, 2)
62
63        stream.consume() # r
64        self.failUnlessEqual(stream.index(), 7)
65        self.failUnlessEqual(stream.charPositionInLine, 3)
66        self.failUnlessEqual(stream.line, 2)
67
68        stream.consume() # EOF
69        self.failUnlessEqual(stream.index(), 7)
70        self.failUnlessEqual(stream.charPositionInLine, 3)
71        self.failUnlessEqual(stream.line, 2)
72
73        stream.consume() # EOF
74        self.failUnlessEqual(stream.index(), 7)
75        self.failUnlessEqual(stream.charPositionInLine, 3)
76        self.failUnlessEqual(stream.line, 2)
77
78
79    def testReset(self):
80        """StringStream.reset()"""
81
82        stream = antlr3.StringStream('foo')
83
84        stream.consume()
85        stream.consume()
86
87        stream.reset()
88        self.failUnlessEqual(stream.index(), 0)
89        self.failUnlessEqual(stream.line, 1)
90        self.failUnlessEqual(stream.charPositionInLine, 0)
91        self.failUnlessEqual(stream.LT(1), 'f')
92
93
94    def testLA(self):
95        """StringStream.LA()"""
96
97        stream = antlr3.StringStream('foo')
98
99        self.failUnlessEqual(stream.LT(1), 'f')
100        self.failUnlessEqual(stream.LT(2), 'o')
101        self.failUnlessEqual(stream.LT(3), 'o')
102
103        stream.consume()
104        stream.consume()
105
106        self.failUnlessEqual(stream.LT(1), 'o')
107        self.failUnlessEqual(stream.LT(2), antlr3.EOF)
108        self.failUnlessEqual(stream.LT(3), antlr3.EOF)
109
110
111    def testSubstring(self):
112        """StringStream.substring()"""
113
114        stream = antlr3.StringStream('foobar')
115
116        self.failUnlessEqual(stream.substring(0, 0), 'f')
117        self.failUnlessEqual(stream.substring(0, 1), 'fo')
118        self.failUnlessEqual(stream.substring(0, 5), 'foobar')
119        self.failUnlessEqual(stream.substring(3, 5), 'bar')
120
121
122    def testSeekForward(self):
123        """StringStream.seek(): forward"""
124
125        stream = antlr3.StringStream('foo\nbar')
126
127        stream.seek(4)
128
129        self.failUnlessEqual(stream.index(), 4)
130        self.failUnlessEqual(stream.line, 2)
131        self.failUnlessEqual(stream.charPositionInLine, 0)
132        self.failUnlessEqual(stream.LT(1), 'b')
133
134
135##     # not yet implemented
136##     def testSeekBackward(self):
137##         """StringStream.seek(): backward"""
138
139##         stream = antlr3.StringStream('foo\nbar')
140
141##         stream.seek(4)
142##         stream.seek(1)
143
144##         self.failUnlessEqual(stream.index(), 1)
145##         self.failUnlessEqual(stream.line, 1)
146##         self.failUnlessEqual(stream.charPositionInLine, 1)
147##         self.failUnlessEqual(stream.LA(1), 'o')
148
149
150    def testMark(self):
151        """StringStream.mark()"""
152
153        stream = antlr3.StringStream('foo\nbar')
154
155        stream.seek(4)
156
157        marker = stream.mark()
158        self.failUnlessEqual(marker, 1)
159        self.failUnlessEqual(stream.markDepth, 1)
160
161        stream.consume()
162        marker = stream.mark()
163        self.failUnlessEqual(marker, 2)
164        self.failUnlessEqual(stream.markDepth, 2)
165
166
167    def testReleaseLast(self):
168        """StringStream.release(): last marker"""
169
170        stream = antlr3.StringStream('foo\nbar')
171
172        stream.seek(4)
173        marker1 = stream.mark()
174
175        stream.consume()
176        marker2 = stream.mark()
177
178        stream.release()
179        self.failUnlessEqual(stream.markDepth, 1)
180
181        # release same marker again, nothing has changed
182        stream.release()
183        self.failUnlessEqual(stream.markDepth, 1)
184
185
186    def testReleaseNested(self):
187        """StringStream.release(): nested"""
188
189        stream = antlr3.StringStream('foo\nbar')
190
191        stream.seek(4)
192        marker1 = stream.mark()
193
194        stream.consume()
195        marker2 = stream.mark()
196
197        stream.consume()
198        marker3 = stream.mark()
199
200        stream.release(marker2)
201        self.failUnlessEqual(stream.markDepth, 1)
202
203
204    def testRewindLast(self):
205        """StringStream.rewind(): last marker"""
206
207        stream = antlr3.StringStream('foo\nbar')
208
209        stream.seek(4)
210
211        marker = stream.mark()
212        stream.consume()
213        stream.consume()
214
215        stream.rewind()
216        self.failUnlessEqual(stream.markDepth, 0)
217        self.failUnlessEqual(stream.index(), 4)
218        self.failUnlessEqual(stream.line, 2)
219        self.failUnlessEqual(stream.charPositionInLine, 0)
220        self.failUnlessEqual(stream.LT(1), 'b')
221
222
223    def testRewindNested(self):
224        """StringStream.rewind(): nested"""
225
226        stream = antlr3.StringStream('foo\nbar')
227
228        stream.seek(4)
229        marker1 = stream.mark()
230
231        stream.consume()
232        marker2 = stream.mark()
233
234        stream.consume()
235        marker3 = stream.mark()
236
237        stream.rewind(marker2)
238        self.failUnlessEqual(stream.markDepth, 1)
239        self.failUnlessEqual(stream.index(), 5)
240        self.failUnlessEqual(stream.line, 2)
241        self.failUnlessEqual(stream.charPositionInLine, 1)
242        self.failUnlessEqual(stream.LT(1), 'a')
243
244
245class TestFileStream(unittest.TestCase):
246    """Test case for the FileStream class."""
247
248
249    def testNoEncoding(self):
250        path = os.path.join(os.path.dirname(__file__), 'teststreams.input1')
251
252        stream = antlr3.FileStream(path)
253
254        stream.seek(4)
255        marker1 = stream.mark()
256
257        stream.consume()
258        marker2 = stream.mark()
259
260        stream.consume()
261        marker3 = stream.mark()
262
263        stream.rewind(marker2)
264        self.failUnlessEqual(stream.markDepth, 1)
265        self.failUnlessEqual(stream.index(), 5)
266        self.failUnlessEqual(stream.line, 2)
267        self.failUnlessEqual(stream.charPositionInLine, 1)
268        self.failUnlessEqual(stream.LT(1), 'a')
269        self.failUnlessEqual(stream.LA(1), ord('a'))
270
271
272    def testEncoded(self):
273        path = os.path.join(os.path.dirname(__file__), 'teststreams.input2')
274
275        stream = antlr3.FileStream(path, 'utf-8')
276
277        stream.seek(4)
278        marker1 = stream.mark()
279
280        stream.consume()
281        marker2 = stream.mark()
282
283        stream.consume()
284        marker3 = stream.mark()
285
286        stream.rewind(marker2)
287        self.failUnlessEqual(stream.markDepth, 1)
288        self.failUnlessEqual(stream.index(), 5)
289        self.failUnlessEqual(stream.line, 2)
290        self.failUnlessEqual(stream.charPositionInLine, 1)
291        self.failUnlessEqual(stream.LT(1), u'ä')
292        self.failUnlessEqual(stream.LA(1), ord(u'ä'))
293
294
295
296class TestInputStream(unittest.TestCase):
297    """Test case for the InputStream class."""
298
299    def testNoEncoding(self):
300        file = StringIO('foo\nbar')
301
302        stream = antlr3.InputStream(file)
303
304        stream.seek(4)
305        marker1 = stream.mark()
306
307        stream.consume()
308        marker2 = stream.mark()
309
310        stream.consume()
311        marker3 = stream.mark()
312
313        stream.rewind(marker2)
314        self.failUnlessEqual(stream.markDepth, 1)
315        self.failUnlessEqual(stream.index(), 5)
316        self.failUnlessEqual(stream.line, 2)
317        self.failUnlessEqual(stream.charPositionInLine, 1)
318        self.failUnlessEqual(stream.LT(1), 'a')
319        self.failUnlessEqual(stream.LA(1), ord('a'))
320
321
322    def testEncoded(self):
323        file = StringIO(u'foo\nbär'.encode('utf-8'))
324
325        stream = antlr3.InputStream(file, 'utf-8')
326
327        stream.seek(4)
328        marker1 = stream.mark()
329
330        stream.consume()
331        marker2 = stream.mark()
332
333        stream.consume()
334        marker3 = stream.mark()
335
336        stream.rewind(marker2)
337        self.failUnlessEqual(stream.markDepth, 1)
338        self.failUnlessEqual(stream.index(), 5)
339        self.failUnlessEqual(stream.line, 2)
340        self.failUnlessEqual(stream.charPositionInLine, 1)
341        self.failUnlessEqual(stream.LT(1), u'ä')
342        self.failUnlessEqual(stream.LA(1), ord(u'ä'))
343
344
345class TestCommonTokenStream(unittest.TestCase):
346    """Test case for the StringStream class."""
347
348    def setUp(self):
349        """Setup test fixure
350
351        The constructor of CommonTokenStream needs a token source. This
352        is a simple mock class providing just the nextToken() method.
353
354        """
355
356        class MockSource(object):
357            def __init__(self):
358                self.tokens = []
359
360            def makeEOFToken(self):
361                return antlr3.CommonToken(type=antlr3.EOF)
362
363            def nextToken(self):
364                try:
365                    return self.tokens.pop(0)
366                except IndexError:
367                    return None
368
369        self.source = MockSource()
370
371
372    def testInit(self):
373        """CommonTokenStream.__init__()"""
374
375        stream = antlr3.CommonTokenStream(self.source)
376        self.failUnlessEqual(stream.index(), -1)
377
378
379    def testSetTokenSource(self):
380        """CommonTokenStream.setTokenSource()"""
381
382        stream = antlr3.CommonTokenStream(None)
383        stream.setTokenSource(self.source)
384        self.failUnlessEqual(stream.index(), -1)
385        self.failUnlessEqual(stream.channel, antlr3.DEFAULT_CHANNEL)
386
387
388    def testLTEmptySource(self):
389        """CommonTokenStream.LT(): EOF (empty source)"""
390
391        stream = antlr3.CommonTokenStream(self.source)
392
393        lt1 = stream.LT(1)
394        self.failUnlessEqual(lt1.type, antlr3.EOF)
395
396
397    def testLT1(self):
398        """CommonTokenStream.LT(1)"""
399
400        self.source.tokens.append(
401            antlr3.CommonToken(type=12)
402            )
403
404        stream = antlr3.CommonTokenStream(self.source)
405
406        lt1 = stream.LT(1)
407        self.failUnlessEqual(lt1.type, 12)
408
409
410    def testLT1WithHidden(self):
411        """CommonTokenStream.LT(1): with hidden tokens"""
412
413        self.source.tokens.append(
414            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
415            )
416
417        self.source.tokens.append(
418            antlr3.CommonToken(type=13)
419            )
420
421        stream = antlr3.CommonTokenStream(self.source)
422
423        lt1 = stream.LT(1)
424        self.failUnlessEqual(lt1.type, 13)
425
426
427    def testLT2BeyondEnd(self):
428        """CommonTokenStream.LT(2): beyond end"""
429
430        self.source.tokens.append(
431            antlr3.CommonToken(type=12)
432            )
433
434        self.source.tokens.append(
435            antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL)
436            )
437
438        stream = antlr3.CommonTokenStream(self.source)
439
440        lt1 = stream.LT(2)
441        self.failUnlessEqual(lt1.type, antlr3.EOF)
442
443
444    # not yet implemented
445    def testLTNegative(self):
446        """CommonTokenStream.LT(-1): look back"""
447
448        self.source.tokens.append(
449            antlr3.CommonToken(type=12)
450            )
451
452        self.source.tokens.append(
453            antlr3.CommonToken(type=13)
454            )
455
456        stream = antlr3.CommonTokenStream(self.source)
457        stream.fillBuffer()
458        stream.consume()
459
460        lt1 = stream.LT(-1)
461        self.failUnlessEqual(lt1.type, 12)
462
463
464    def testLB1(self):
465        """CommonTokenStream.LB(1)"""
466
467        self.source.tokens.append(
468            antlr3.CommonToken(type=12)
469            )
470
471        self.source.tokens.append(
472            antlr3.CommonToken(type=13)
473            )
474
475        stream = antlr3.CommonTokenStream(self.source)
476        stream.fillBuffer()
477        stream.consume()
478
479        self.failUnlessEqual(stream.LB(1).type, 12)
480
481
482    def testLTZero(self):
483        """CommonTokenStream.LT(0)"""
484
485        self.source.tokens.append(
486            antlr3.CommonToken(type=12)
487            )
488
489        self.source.tokens.append(
490            antlr3.CommonToken(type=13)
491            )
492
493        stream = antlr3.CommonTokenStream(self.source)
494
495        lt1 = stream.LT(0)
496        self.failUnless(lt1 is None)
497
498
499    def testLBBeyondBegin(self):
500        """CommonTokenStream.LB(-1): beyond begin"""
501
502        self.source.tokens.append(
503            antlr3.CommonToken(type=12)
504            )
505
506        self.source.tokens.append(
507            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
508            )
509
510        self.source.tokens.append(
511            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
512            )
513
514        self.source.tokens.append(
515            antlr3.CommonToken(type=13)
516            )
517
518        stream = antlr3.CommonTokenStream(self.source)
519        self.failUnless(stream.LB(1) is None)
520
521        stream.consume()
522        stream.consume()
523        self.failUnless(stream.LB(3) is None)
524
525
526    def testFillBuffer(self):
527        """CommonTokenStream.fillBuffer()"""
528
529        self.source.tokens.append(
530            antlr3.CommonToken(type=12)
531            )
532
533        self.source.tokens.append(
534            antlr3.CommonToken(type=13)
535            )
536
537        self.source.tokens.append(
538            antlr3.CommonToken(type=14)
539            )
540
541        self.source.tokens.append(
542            antlr3.CommonToken(type=antlr3.EOF)
543            )
544
545        stream = antlr3.CommonTokenStream(self.source)
546        stream.fillBuffer()
547
548        self.failUnlessEqual(len(stream.tokens), 3)
549        self.failUnlessEqual(stream.tokens[0].type, 12)
550        self.failUnlessEqual(stream.tokens[1].type, 13)
551        self.failUnlessEqual(stream.tokens[2].type, 14)
552
553
554    def testConsume(self):
555        """CommonTokenStream.consume()"""
556
557        self.source.tokens.append(
558            antlr3.CommonToken(type=12)
559            )
560
561        self.source.tokens.append(
562            antlr3.CommonToken(type=13)
563            )
564
565        self.source.tokens.append(
566            antlr3.CommonToken(type=antlr3.EOF)
567            )
568
569        stream = antlr3.CommonTokenStream(self.source)
570        self.failUnlessEqual(stream.LA(1), 12)
571
572        stream.consume()
573        self.failUnlessEqual(stream.LA(1), 13)
574
575        stream.consume()
576        self.failUnlessEqual(stream.LA(1), antlr3.EOF)
577
578        stream.consume()
579        self.failUnlessEqual(stream.LA(1), antlr3.EOF)
580
581
582    def testSeek(self):
583        """CommonTokenStream.seek()"""
584
585        self.source.tokens.append(
586            antlr3.CommonToken(type=12)
587            )
588
589        self.source.tokens.append(
590            antlr3.CommonToken(type=13)
591            )
592
593        self.source.tokens.append(
594            antlr3.CommonToken(type=antlr3.EOF)
595            )
596
597        stream = antlr3.CommonTokenStream(self.source)
598        self.failUnlessEqual(stream.LA(1), 12)
599
600        stream.seek(2)
601        self.failUnlessEqual(stream.LA(1), antlr3.EOF)
602
603        stream.seek(0)
604        self.failUnlessEqual(stream.LA(1), 12)
605
606
607    def testMarkRewind(self):
608        """CommonTokenStream.mark()/rewind()"""
609
610        self.source.tokens.append(
611            antlr3.CommonToken(type=12)
612            )
613
614        self.source.tokens.append(
615            antlr3.CommonToken(type=13)
616            )
617
618        self.source.tokens.append(
619            antlr3.CommonToken(type=antlr3.EOF)
620            )
621
622        stream = antlr3.CommonTokenStream(self.source)
623        stream.fillBuffer()
624
625        stream.consume()
626        marker = stream.mark()
627
628        stream.consume()
629        stream.rewind(marker)
630
631        self.failUnlessEqual(stream.LA(1), 13)
632
633
634    def testToString(self):
635        """CommonTokenStream.toString()"""
636
637        self.source.tokens.append(
638            antlr3.CommonToken(type=12, text="foo")
639            )
640
641        self.source.tokens.append(
642            antlr3.CommonToken(type=13, text="bar")
643            )
644
645        self.source.tokens.append(
646            antlr3.CommonToken(type=14, text="gnurz")
647            )
648
649        self.source.tokens.append(
650            antlr3.CommonToken(type=15, text="blarz")
651            )
652
653        stream = antlr3.CommonTokenStream(self.source)
654
655        assert stream.toString() == "foobargnurzblarz"
656        assert stream.toString(1, 2) == "bargnurz"
657        assert stream.toString(stream.tokens[1], stream.tokens[-2]) == "bargnurz"
658
659
660if __name__ == "__main__":
661    unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
662