• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import unittest
2import textwrap
3import antlr3
4import antlr3.tree
5import testbase
6
7class T(testbase.ANTLRTest):
8    def walkerClass(self, base):
9        class TWalker(base):
10            def __init__(self, *args, **kwargs):
11                super().__init__(*args, **kwargs)
12
13                self._output = ""
14
15
16            def capture(self, t):
17                self._output += t
18
19
20            def traceIn(self, ruleName, ruleIndex):
21                self.traces.append('>'+ruleName)
22
23
24            def traceOut(self, ruleName, ruleIndex):
25                self.traces.append('<'+ruleName)
26
27
28            def recover(self, input, re):
29                # no error recovery yet, just crash!
30                raise
31
32        return TWalker
33
34
35    def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input):
36        lexerCls, parserCls = self.compileInlineGrammar(grammar)
37        walkerCls = self.compileInlineGrammar(treeGrammar)
38
39        cStream = antlr3.StringStream(input)
40        lexer = lexerCls(cStream)
41        tStream = antlr3.CommonTokenStream(lexer)
42        parser = parserCls(tStream)
43        r = getattr(parser, grammarEntry)()
44        nodes = antlr3.tree.CommonTreeNodeStream(r.tree)
45        nodes.setTokenStream(tStream)
46        walker = walkerCls(nodes)
47        getattr(walker, treeEntry)()
48
49        return walker._output
50
51
52    def testFlatList(self):
53        grammar = textwrap.dedent(
54        r'''grammar T;
55        options {
56            language=Python3;
57            output=AST;
58        }
59        a : ID INT;
60        ID : 'a'..'z'+ ;
61        INT : '0'..'9'+;
62        WS : (' '|'\n') {$channel=HIDDEN;} ;
63        ''')
64
65        treeGrammar = textwrap.dedent(
66        r'''tree grammar TP;
67        options {
68            language=Python3;
69            ASTLabelType=CommonTree;
70        }
71        a : ID INT
72            {self.capture("{}, {}".format($ID, $INT))}
73          ;
74        ''')
75
76        found = self.execTreeParser(
77            grammar, 'a',
78            treeGrammar, 'a',
79            "abc 34"
80            )
81
82        self.assertEqual("abc, 34", found)
83
84
85
86    def testSimpleTree(self):
87        grammar = textwrap.dedent(
88            r'''grammar T;
89            options {
90                language=Python3;
91                output=AST;
92            }
93            a : ID INT -> ^(ID INT);
94            ID : 'a'..'z'+ ;
95            INT : '0'..'9'+;
96            WS : (' '|'\\n') {$channel=HIDDEN;} ;
97            ''')
98
99        treeGrammar = textwrap.dedent(
100            r'''tree grammar TP;
101            options {
102                language=Python3;
103                ASTLabelType=CommonTree;
104            }
105            a : ^(ID INT)
106                {self.capture(str($ID)+", "+str($INT))}
107              ;
108            ''')
109
110        found = self.execTreeParser(
111            grammar, 'a',
112            treeGrammar, 'a',
113            "abc 34"
114            )
115
116        self.assertEqual("abc, 34", found)
117
118
119    def testFlatVsTreeDecision(self):
120        grammar = textwrap.dedent(
121            r'''grammar T;
122            options {
123                language=Python3;
124                output=AST;
125            }
126            a : b c ;
127            b : ID INT -> ^(ID INT);
128            c : ID INT;
129            ID : 'a'..'z'+ ;
130            INT : '0'..'9'+;
131            WS : (' '|'\\n') {$channel=HIDDEN;} ;
132            ''')
133
134        treeGrammar = textwrap.dedent(
135            r'''tree grammar TP;
136            options {
137                language=Python3;
138                ASTLabelType=CommonTree;
139            }
140            a : b b ;
141            b : ID INT    {self.capture(str($ID)+" "+str($INT)+'\n')}
142              | ^(ID INT) {self.capture("^("+str($ID)+" "+str($INT)+')');}
143              ;
144            ''')
145
146        found = self.execTreeParser(
147            grammar, 'a',
148            treeGrammar, 'a',
149            "a 1 b 2"
150            )
151        self.assertEqual("^(a 1)b 2\n", found)
152
153
154    def testFlatVsTreeDecision2(self):
155        grammar = textwrap.dedent(
156            r"""grammar T;
157            options {
158                language=Python3;
159                output=AST;
160            }
161            a : b c ;
162            b : ID INT+ -> ^(ID INT+);
163            c : ID INT+;
164            ID : 'a'..'z'+ ;
165            INT : '0'..'9'+;
166            WS : (' '|'\n') {$channel=HIDDEN;} ;
167            """)
168
169        treeGrammar = textwrap.dedent(
170            r'''tree grammar TP;
171            options {
172                language=Python3;
173                ASTLabelType=CommonTree;
174            }
175            a : b b ;
176            b : ID INT+    {self.capture(str($ID)+" "+str($INT)+"\n")}
177              | ^(x=ID (y=INT)+) {self.capture("^("+str($x)+' '+str($y)+')')}
178              ;
179            ''')
180
181        found = self.execTreeParser(
182            grammar, 'a',
183            treeGrammar, 'a',
184            "a 1 2 3 b 4 5"
185            )
186        self.assertEqual("^(a 3)b 5\n", found)
187
188
189    def testCyclicDFALookahead(self):
190        grammar = textwrap.dedent(
191            r'''grammar T;
192            options {
193                language=Python3;
194                output=AST;
195            }
196            a : ID INT+ PERIOD;
197            ID : 'a'..'z'+ ;
198            INT : '0'..'9'+;
199            SEMI : ';' ;
200            PERIOD : '.' ;
201            WS : (' '|'\n') {$channel=HIDDEN;} ;
202            ''')
203
204        treeGrammar = textwrap.dedent(
205            r'''tree grammar TP;
206            options {
207                language=Python3;
208                ASTLabelType=CommonTree;
209            }
210            a : ID INT+ PERIOD {self.capture("alt 1")}
211              | ID INT+ SEMI   {self.capture("alt 2")}
212              ;
213            ''')
214
215        found = self.execTreeParser(
216            grammar, 'a',
217            treeGrammar, 'a',
218            "a 1 2 3."
219            )
220        self.assertEqual("alt 1", found)
221
222
223    def testNullableChildList(self):
224        grammar = textwrap.dedent(
225            r'''grammar T;
226            options {
227                language=Python3;
228                output=AST;
229            }
230            a : ID INT? -> ^(ID INT?);
231            ID : 'a'..'z'+ ;
232            INT : '0'..'9'+;
233            WS : (' '|'\\n') {$channel=HIDDEN;} ;
234            ''')
235
236        treeGrammar = textwrap.dedent(
237            r'''tree grammar TP;
238            options {
239                language=Python3;
240                ASTLabelType=CommonTree;
241            }
242            a : ^(ID INT?)
243                {self.capture(str($ID))}
244              ;
245            ''')
246
247        found = self.execTreeParser(
248            grammar, 'a',
249            treeGrammar, 'a',
250            "abc"
251            )
252        self.assertEqual("abc", found)
253
254
255    def testNullableChildList2(self):
256        grammar = textwrap.dedent(
257            r'''grammar T;
258            options {
259                language=Python3;
260                output=AST;
261            }
262            a : ID INT? SEMI -> ^(ID INT?) SEMI ;
263            ID : 'a'..'z'+ ;
264            INT : '0'..'9'+;
265            SEMI : ';' ;
266            WS : (' '|'\n') {$channel=HIDDEN;} ;
267            ''')
268
269        treeGrammar = textwrap.dedent(
270            r'''tree grammar TP;
271            options {
272                language=Python3;
273                ASTLabelType=CommonTree;
274            }
275            a : ^(ID INT?) SEMI
276                {self.capture(str($ID))}
277              ;
278            ''')
279
280        found = self.execTreeParser(
281            grammar, 'a',
282            treeGrammar, 'a',
283            "abc;"
284            )
285        self.assertEqual("abc", found)
286
287
288    def testNullableChildList3(self):
289        grammar = textwrap.dedent(
290            r'''grammar T;
291            options {
292                language=Python3;
293                output=AST;
294            }
295            a : x=ID INT? (y=ID)? SEMI -> ^($x INT? $y?) SEMI ;
296            ID : 'a'..'z'+ ;
297            INT : '0'..'9'+;
298            SEMI : ';' ;
299            WS : (' '|'\\n') {$channel=HIDDEN;} ;
300            ''')
301
302        treeGrammar = textwrap.dedent(
303            r'''tree grammar TP;
304            options {
305                language=Python3;
306                ASTLabelType=CommonTree;
307            }
308            a : ^(ID INT? b) SEMI
309                {self.capture(str($ID)+", "+str($b.text))}
310              ;
311            b : ID? ;
312            ''')
313
314        found = self.execTreeParser(
315            grammar, 'a',
316            treeGrammar, 'a',
317            "abc def;"
318            )
319        self.assertEqual("abc, def", found)
320
321
322    def testActionsAfterRoot(self):
323        grammar = textwrap.dedent(
324            r'''grammar T;
325            options {
326                language=Python3;
327                output=AST;
328            }
329            a : x=ID INT? SEMI -> ^($x INT?) ;
330            ID : 'a'..'z'+ ;
331            INT : '0'..'9'+;
332            SEMI : ';' ;
333            WS : (' '|'\n') {$channel=HIDDEN;} ;
334            ''')
335
336        treeGrammar = textwrap.dedent(
337            r'''tree grammar TP;
338            options {
339                language=Python3;
340                ASTLabelType=CommonTree;
341            }
342            a @init {x=0} : ^(ID {x=1} {x=2} INT?)
343                {self.capture(str($ID)+", "+str(x))}
344              ;
345            ''')
346
347        found = self.execTreeParser(
348            grammar, 'a',
349            treeGrammar, 'a',
350            "abc;"
351            )
352        self.assertEqual("abc, 2", found)
353
354
355    def testWildcardLookahead(self):
356        grammar = textwrap.dedent(
357            r'''
358            grammar T;
359            options {language=Python3; output=AST;}
360            a : ID '+'^ INT;
361            ID : 'a'..'z'+ ;
362            INT : '0'..'9'+;
363            SEMI : ';' ;
364            PERIOD : '.' ;
365            WS : (' '|'\n') {$channel=HIDDEN;} ;
366            ''')
367
368        treeGrammar = textwrap.dedent(
369            r'''
370            tree grammar TP;
371            options {language=Python3; tokenVocab=T; ASTLabelType=CommonTree;}
372            a : ^('+' . INT) { self.capture("alt 1") }
373              ;
374            ''')
375
376        found = self.execTreeParser(
377            grammar, 'a',
378            treeGrammar, 'a',
379            "a + 2")
380        self.assertEqual("alt 1", found)
381
382
383    def testWildcardLookahead2(self):
384        grammar = textwrap.dedent(
385            r'''
386            grammar T;
387            options {language=Python3; output=AST;}
388            a : ID '+'^ INT;
389            ID : 'a'..'z'+ ;
390            INT : '0'..'9'+;
391            SEMI : ';' ;
392            PERIOD : '.' ;
393            WS : (' '|'\n') {$channel=HIDDEN;} ;
394            ''')
395
396        treeGrammar = textwrap.dedent(
397            r'''
398            tree grammar TP;
399            options {language=Python3; tokenVocab=T; ASTLabelType=CommonTree;}
400            a : ^('+' . INT) { self.capture("alt 1") }
401              | ^('+' . .)   { self.capture("alt 2") }
402              ;
403            ''')
404
405        # AMBIG upon '+' DOWN INT UP etc.. but so what.
406
407        found = self.execTreeParser(
408            grammar, 'a',
409            treeGrammar, 'a',
410            "a + 2")
411        self.assertEqual("alt 1", found)
412
413
414    def testWildcardLookahead3(self):
415        grammar = textwrap.dedent(
416            r'''
417            grammar T;
418            options {language=Python3; output=AST;}
419            a : ID '+'^ INT;
420            ID : 'a'..'z'+ ;
421            INT : '0'..'9'+;
422            SEMI : ';' ;
423            PERIOD : '.' ;
424            WS : (' '|'\n') {$channel=HIDDEN;} ;
425            ''')
426
427        treeGrammar = textwrap.dedent(
428            r'''
429            tree grammar TP;
430            options {language=Python3; tokenVocab=T; ASTLabelType=CommonTree;}
431            a : ^('+' ID INT) { self.capture("alt 1") }
432              | ^('+' . .)   { self.capture("alt 2") }
433              ;
434            ''')
435
436        # AMBIG upon '+' DOWN INT UP etc.. but so what.
437
438        found = self.execTreeParser(
439            grammar, 'a',
440            treeGrammar, 'a',
441            "a + 2")
442        self.assertEqual("alt 1", found)
443
444
445    def testWildcardPlusLookahead(self):
446        grammar = textwrap.dedent(
447            r'''
448            grammar T;
449            options {language=Python3; output=AST;}
450            a : ID '+'^ INT;
451            ID : 'a'..'z'+ ;
452            INT : '0'..'9'+;
453            SEMI : ';' ;
454            PERIOD : '.' ;
455            WS : (' '|'\n') {$channel=HIDDEN;} ;
456            ''')
457
458        treeGrammar = textwrap.dedent(
459            r'''
460            tree grammar TP;
461            options {language=Python3; tokenVocab=T; ASTLabelType=CommonTree;}
462            a : ^('+' INT INT ) { self.capture("alt 1") }
463              | ^('+' .+)   { self.capture("alt 2") }
464              ;
465            ''')
466
467        # AMBIG upon '+' DOWN INT UP etc.. but so what.
468
469        found = self.execTreeParser(
470            grammar, 'a',
471            treeGrammar, 'a',
472            "a + 2")
473        self.assertEqual("alt 2", found)
474
475
476if __name__ == '__main__':
477    unittest.main()
478