• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1
2from io import StringIO
3import os
4import unittest
5import antlr3
6
7
8class TestStringStream(unittest.TestCase):
9    """Test case for the StringStream class."""
10
11    def testSize(self):
12        """StringStream.size()"""
13
14        stream = antlr3.StringStream('foo')
15
16        self.assertEqual(stream.size(), 3)
17
18
19    def testIndex(self):
20        """StringStream.index()"""
21
22        stream = antlr3.StringStream('foo')
23
24        self.assertEqual(stream.index(), 0)
25
26
27    def testConsume(self):
28        """StringStream.consume()"""
29
30        stream = antlr3.StringStream('foo\nbar')
31
32        stream.consume() # f
33        self.assertEqual(stream.index(), 1)
34        self.assertEqual(stream.charPositionInLine, 1)
35        self.assertEqual(stream.line, 1)
36
37        stream.consume() # o
38        self.assertEqual(stream.index(), 2)
39        self.assertEqual(stream.charPositionInLine, 2)
40        self.assertEqual(stream.line, 1)
41
42        stream.consume() # o
43        self.assertEqual(stream.index(), 3)
44        self.assertEqual(stream.charPositionInLine, 3)
45        self.assertEqual(stream.line, 1)
46
47        stream.consume() # \n
48        self.assertEqual(stream.index(), 4)
49        self.assertEqual(stream.charPositionInLine, 0)
50        self.assertEqual(stream.line, 2)
51
52        stream.consume() # b
53        self.assertEqual(stream.index(), 5)
54        self.assertEqual(stream.charPositionInLine, 1)
55        self.assertEqual(stream.line, 2)
56
57        stream.consume() # a
58        self.assertEqual(stream.index(), 6)
59        self.assertEqual(stream.charPositionInLine, 2)
60        self.assertEqual(stream.line, 2)
61
62        stream.consume() # r
63        self.assertEqual(stream.index(), 7)
64        self.assertEqual(stream.charPositionInLine, 3)
65        self.assertEqual(stream.line, 2)
66
67        stream.consume() # EOF
68        self.assertEqual(stream.index(), 7)
69        self.assertEqual(stream.charPositionInLine, 3)
70        self.assertEqual(stream.line, 2)
71
72        stream.consume() # EOF
73        self.assertEqual(stream.index(), 7)
74        self.assertEqual(stream.charPositionInLine, 3)
75        self.assertEqual(stream.line, 2)
76
77
78    def testReset(self):
79        """StringStream.reset()"""
80
81        stream = antlr3.StringStream('foo')
82
83        stream.consume()
84        stream.consume()
85
86        stream.reset()
87        self.assertEqual(stream.index(), 0)
88        self.assertEqual(stream.line, 1)
89        self.assertEqual(stream.charPositionInLine, 0)
90        self.assertEqual(stream.LT(1), 'f')
91
92
93    def testLA(self):
94        """StringStream.LA()"""
95
96        stream = antlr3.StringStream('foo')
97
98        self.assertEqual(stream.LT(1), 'f')
99        self.assertEqual(stream.LT(2), 'o')
100        self.assertEqual(stream.LT(3), 'o')
101
102        stream.consume()
103        stream.consume()
104
105        self.assertEqual(stream.LT(1), 'o')
106        self.assertEqual(stream.LT(2), antlr3.EOF)
107        self.assertEqual(stream.LT(3), antlr3.EOF)
108
109
110    def testSubstring(self):
111        """StringStream.substring()"""
112
113        stream = antlr3.StringStream('foobar')
114
115        self.assertEqual(stream.substring(0, 0), 'f')
116        self.assertEqual(stream.substring(0, 1), 'fo')
117        self.assertEqual(stream.substring(0, 5), 'foobar')
118        self.assertEqual(stream.substring(3, 5), 'bar')
119
120
121    def testSeekForward(self):
122        """StringStream.seek(): forward"""
123
124        stream = antlr3.StringStream('foo\nbar')
125
126        stream.seek(4)
127
128        self.assertEqual(stream.index(), 4)
129        self.assertEqual(stream.line, 2)
130        self.assertEqual(stream.charPositionInLine, 0)
131        self.assertEqual(stream.LT(1), 'b')
132
133
134##     # not yet implemented
135##     def testSeekBackward(self):
136##         """StringStream.seek(): backward"""
137
138##         stream = antlr3.StringStream('foo\nbar')
139
140##         stream.seek(4)
141##         stream.seek(1)
142
143##         self.assertEqual(stream.index(), 1)
144##         self.assertEqual(stream.line, 1)
145##         self.assertEqual(stream.charPositionInLine, 1)
146##         self.assertEqual(stream.LA(1), 'o')
147
148
149    def testMark(self):
150        """StringStream.mark()"""
151
152        stream = antlr3.StringStream('foo\nbar')
153
154        stream.seek(4)
155
156        marker = stream.mark()
157        self.assertEqual(marker, 1)
158        self.assertEqual(stream.markDepth, 1)
159
160        stream.consume()
161        marker = stream.mark()
162        self.assertEqual(marker, 2)
163        self.assertEqual(stream.markDepth, 2)
164
165
166    def testReleaseLast(self):
167        """StringStream.release(): last marker"""
168
169        stream = antlr3.StringStream('foo\nbar')
170
171        stream.seek(4)
172        marker1 = stream.mark()
173
174        stream.consume()
175        marker2 = stream.mark()
176
177        stream.release()
178        self.assertEqual(stream.markDepth, 1)
179
180        # release same marker again, nothing has changed
181        stream.release()
182        self.assertEqual(stream.markDepth, 1)
183
184
185    def testReleaseNested(self):
186        """StringStream.release(): nested"""
187
188        stream = antlr3.StringStream('foo\nbar')
189
190        stream.seek(4)
191        marker1 = stream.mark()
192
193        stream.consume()
194        marker2 = stream.mark()
195
196        stream.consume()
197        marker3 = stream.mark()
198
199        stream.release(marker2)
200        self.assertEqual(stream.markDepth, 1)
201
202
203    def testRewindLast(self):
204        """StringStream.rewind(): last marker"""
205
206        stream = antlr3.StringStream('foo\nbar')
207
208        stream.seek(4)
209
210        marker = stream.mark()
211        stream.consume()
212        stream.consume()
213
214        stream.rewind()
215        self.assertEqual(stream.markDepth, 0)
216        self.assertEqual(stream.index(), 4)
217        self.assertEqual(stream.line, 2)
218        self.assertEqual(stream.charPositionInLine, 0)
219        self.assertEqual(stream.LT(1), 'b')
220
221
222    def testRewindNested(self):
223        """StringStream.rewind(): nested"""
224
225        stream = antlr3.StringStream('foo\nbar')
226
227        stream.seek(4)
228        marker1 = stream.mark()
229
230        stream.consume()
231        marker2 = stream.mark()
232
233        stream.consume()
234        marker3 = stream.mark()
235
236        stream.rewind(marker2)
237        self.assertEqual(stream.markDepth, 1)
238        self.assertEqual(stream.index(), 5)
239        self.assertEqual(stream.line, 2)
240        self.assertEqual(stream.charPositionInLine, 1)
241        self.assertEqual(stream.LT(1), 'a')
242
243
244class TestFileStream(unittest.TestCase):
245    """Test case for the FileStream class."""
246
247
248    def testNoEncoding(self):
249        path = os.path.join(os.path.dirname(__file__), 'teststreams.input1')
250
251        stream = antlr3.FileStream(path)
252
253        stream.seek(4)
254        marker1 = stream.mark()
255
256        stream.consume()
257        marker2 = stream.mark()
258
259        stream.consume()
260        marker3 = stream.mark()
261
262        stream.rewind(marker2)
263        self.assertEqual(stream.markDepth, 1)
264        self.assertEqual(stream.index(), 5)
265        self.assertEqual(stream.line, 2)
266        self.assertEqual(stream.charPositionInLine, 1)
267        self.assertEqual(stream.LT(1), 'a')
268        self.assertEqual(stream.LA(1), ord('a'))
269
270
271    def testEncoded(self):
272        path = os.path.join(os.path.dirname(__file__), 'teststreams.input2')
273
274        stream = antlr3.FileStream(path)
275
276        stream.seek(4)
277        marker1 = stream.mark()
278
279        stream.consume()
280        marker2 = stream.mark()
281
282        stream.consume()
283        marker3 = stream.mark()
284
285        stream.rewind(marker2)
286        self.assertEqual(stream.markDepth, 1)
287        self.assertEqual(stream.index(), 5)
288        self.assertEqual(stream.line, 2)
289        self.assertEqual(stream.charPositionInLine, 1)
290        self.assertEqual(stream.LT(1), 'ä')
291        self.assertEqual(stream.LA(1), ord('ä'))
292
293
294
295class TestInputStream(unittest.TestCase):
296    """Test case for the InputStream class."""
297
298    def testNoEncoding(self):
299        file = StringIO('foo\nbar')
300
301        stream = antlr3.InputStream(file)
302
303        stream.seek(4)
304        marker1 = stream.mark()
305
306        stream.consume()
307        marker2 = stream.mark()
308
309        stream.consume()
310        marker3 = stream.mark()
311
312        stream.rewind(marker2)
313        self.assertEqual(stream.markDepth, 1)
314        self.assertEqual(stream.index(), 5)
315        self.assertEqual(stream.line, 2)
316        self.assertEqual(stream.charPositionInLine, 1)
317        self.assertEqual(stream.LT(1), 'a')
318        self.assertEqual(stream.LA(1), ord('a'))
319
320
321    def testEncoded(self):
322        file = StringIO('foo\nbär')
323
324        stream = antlr3.InputStream(file)
325
326        stream.seek(4)
327        marker1 = stream.mark()
328
329        stream.consume()
330        marker2 = stream.mark()
331
332        stream.consume()
333        marker3 = stream.mark()
334
335        stream.rewind(marker2)
336        self.assertEqual(stream.markDepth, 1)
337        self.assertEqual(stream.index(), 5)
338        self.assertEqual(stream.line, 2)
339        self.assertEqual(stream.charPositionInLine, 1)
340        self.assertEqual(stream.LT(1), 'ä')
341        self.assertEqual(stream.LA(1), ord('ä'))
342
343
344class TestCommonTokenStream(unittest.TestCase):
345    """Test case for the StringStream class."""
346
347    def setUp(self):
348        """Setup test fixure
349
350        The constructor of CommonTokenStream needs a token source. This
351        is a simple mock class providing just the nextToken() method.
352
353        """
354
355        class MockSource(object):
356            def __init__(self):
357                self.tokens = []
358
359            def makeEOFToken(self):
360                return antlr3.CommonToken(type=antlr3.EOF)
361
362            def nextToken(self):
363                if self.tokens:
364                    return self.tokens.pop(0)
365                return None
366
367        self.source = MockSource()
368
369
370    def testInit(self):
371        """CommonTokenStream.__init__()"""
372
373        stream = antlr3.CommonTokenStream(self.source)
374        self.assertEqual(stream.index(), -1)
375
376
377    def testSetTokenSource(self):
378        """CommonTokenStream.setTokenSource()"""
379
380        stream = antlr3.CommonTokenStream(None)
381        stream.setTokenSource(self.source)
382        self.assertEqual(stream.index(), -1)
383        self.assertEqual(stream.channel, antlr3.DEFAULT_CHANNEL)
384
385
386    def testLTEmptySource(self):
387        """CommonTokenStream.LT(): EOF (empty source)"""
388
389        stream = antlr3.CommonTokenStream(self.source)
390
391        lt1 = stream.LT(1)
392        self.assertEqual(lt1.type, antlr3.EOF)
393
394
395    def testLT1(self):
396        """CommonTokenStream.LT(1)"""
397
398        self.source.tokens.append(
399            antlr3.CommonToken(type=12)
400            )
401
402        stream = antlr3.CommonTokenStream(self.source)
403
404        lt1 = stream.LT(1)
405        self.assertEqual(lt1.type, 12)
406
407
408    def testLT1WithHidden(self):
409        """CommonTokenStream.LT(1): with hidden tokens"""
410
411        self.source.tokens.append(
412            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
413            )
414
415        self.source.tokens.append(
416            antlr3.CommonToken(type=13)
417            )
418
419        stream = antlr3.CommonTokenStream(self.source)
420
421        lt1 = stream.LT(1)
422        self.assertEqual(lt1.type, 13)
423
424
425    def testLT2BeyondEnd(self):
426        """CommonTokenStream.LT(2): beyond end"""
427
428        self.source.tokens.append(
429            antlr3.CommonToken(type=12)
430            )
431
432        self.source.tokens.append(
433            antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL)
434            )
435
436        stream = antlr3.CommonTokenStream(self.source)
437
438        lt1 = stream.LT(2)
439        self.assertEqual(lt1.type, antlr3.EOF)
440
441
442    # not yet implemented
443    def testLTNegative(self):
444        """CommonTokenStream.LT(-1): look back"""
445
446        self.source.tokens.append(
447            antlr3.CommonToken(type=12)
448            )
449
450        self.source.tokens.append(
451            antlr3.CommonToken(type=13)
452            )
453
454        stream = antlr3.CommonTokenStream(self.source)
455        stream.fillBuffer()
456        stream.consume()
457
458        lt1 = stream.LT(-1)
459        self.assertEqual(lt1.type, 12)
460
461
462    def testLB1(self):
463        """CommonTokenStream.LB(1)"""
464
465        self.source.tokens.append(
466            antlr3.CommonToken(type=12)
467            )
468
469        self.source.tokens.append(
470            antlr3.CommonToken(type=13)
471            )
472
473        stream = antlr3.CommonTokenStream(self.source)
474        stream.fillBuffer()
475        stream.consume()
476
477        self.assertEqual(stream.LB(1).type, 12)
478
479
480    def testLTZero(self):
481        """CommonTokenStream.LT(0)"""
482
483        self.source.tokens.append(
484            antlr3.CommonToken(type=12)
485            )
486
487        self.source.tokens.append(
488            antlr3.CommonToken(type=13)
489            )
490
491        stream = antlr3.CommonTokenStream(self.source)
492
493        lt1 = stream.LT(0)
494        self.assertIsNone(lt1)
495
496
497    def testLBBeyondBegin(self):
498        """CommonTokenStream.LB(-1): beyond begin"""
499
500        self.source.tokens.append(
501            antlr3.CommonToken(type=12)
502            )
503
504        self.source.tokens.append(
505            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
506            )
507
508        self.source.tokens.append(
509            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
510            )
511
512        self.source.tokens.append(
513            antlr3.CommonToken(type=13)
514            )
515
516        stream = antlr3.CommonTokenStream(self.source)
517        self.assertIsNone(stream.LB(1))
518
519        stream.consume()
520        stream.consume()
521        self.assertIsNone(stream.LB(3))
522
523
524    def testFillBuffer(self):
525        """CommonTokenStream.fillBuffer()"""
526
527        self.source.tokens.append(
528            antlr3.CommonToken(type=12)
529            )
530
531        self.source.tokens.append(
532            antlr3.CommonToken(type=13)
533            )
534
535        self.source.tokens.append(
536            antlr3.CommonToken(type=14)
537            )
538
539        self.source.tokens.append(
540            antlr3.CommonToken(type=antlr3.EOF)
541            )
542
543        stream = antlr3.CommonTokenStream(self.source)
544        stream.fillBuffer()
545
546        self.assertEqual(len(stream.tokens), 3)
547        self.assertEqual(stream.tokens[0].type, 12)
548        self.assertEqual(stream.tokens[1].type, 13)
549        self.assertEqual(stream.tokens[2].type, 14)
550
551
552    def testConsume(self):
553        """CommonTokenStream.consume()"""
554
555        self.source.tokens.append(
556            antlr3.CommonToken(type=12)
557            )
558
559        self.source.tokens.append(
560            antlr3.CommonToken(type=13)
561            )
562
563        self.source.tokens.append(
564            antlr3.CommonToken(type=antlr3.EOF)
565            )
566
567        stream = antlr3.CommonTokenStream(self.source)
568        self.assertEqual(stream.LA(1), 12)
569
570        stream.consume()
571        self.assertEqual(stream.LA(1), 13)
572
573        stream.consume()
574        self.assertEqual(stream.LA(1), antlr3.EOF)
575
576        stream.consume()
577        self.assertEqual(stream.LA(1), antlr3.EOF)
578
579
580    def testSeek(self):
581        """CommonTokenStream.seek()"""
582
583        self.source.tokens.append(
584            antlr3.CommonToken(type=12)
585            )
586
587        self.source.tokens.append(
588            antlr3.CommonToken(type=13)
589            )
590
591        self.source.tokens.append(
592            antlr3.CommonToken(type=antlr3.EOF)
593            )
594
595        stream = antlr3.CommonTokenStream(self.source)
596        self.assertEqual(stream.LA(1), 12)
597
598        stream.seek(2)
599        self.assertEqual(stream.LA(1), antlr3.EOF)
600
601        stream.seek(0)
602        self.assertEqual(stream.LA(1), 12)
603
604
605    def testMarkRewind(self):
606        """CommonTokenStream.mark()/rewind()"""
607
608        self.source.tokens.append(
609            antlr3.CommonToken(type=12)
610            )
611
612        self.source.tokens.append(
613            antlr3.CommonToken(type=13)
614            )
615
616        self.source.tokens.append(
617            antlr3.CommonToken(type=antlr3.EOF)
618            )
619
620        stream = antlr3.CommonTokenStream(self.source)
621        stream.fillBuffer()
622
623        stream.consume()
624        marker = stream.mark()
625
626        stream.consume()
627        stream.rewind(marker)
628
629        self.assertEqual(stream.LA(1), 13)
630
631
632    def testToString(self):
633        """CommonTokenStream.toString()"""
634
635        self.source.tokens.append(
636            antlr3.CommonToken(type=12, text="foo")
637            )
638
639        self.source.tokens.append(
640            antlr3.CommonToken(type=13, text="bar")
641            )
642
643        self.source.tokens.append(
644            antlr3.CommonToken(type=14, text="gnurz")
645            )
646
647        self.source.tokens.append(
648            antlr3.CommonToken(type=15, text="blarz")
649            )
650
651        stream = antlr3.CommonTokenStream(self.source)
652
653        self.assertEqual(stream.toString(), "foobargnurzblarz")
654        self.assertEqual(stream.toString(1, 2), "bargnurz")
655        self.assertEqual(stream.toString(stream.tokens[1], stream.tokens[-2]), "bargnurz")
656
657
658if __name__ == "__main__":
659    unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
660