• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import unittest
2import textwrap
3import antlr3
4import antlr3.tree
5import testbase
6
7class T(testbase.ANTLRTest):
8    def walkerClass(self, base):
9        class TWalker(base):
10            def __init__(self, *args, **kwargs):
11                base.__init__(self, *args, **kwargs)
12
13                self._output = ""
14
15
16            def capture(self, t):
17                self._output += t
18
19
20            def traceIn(self, ruleName, ruleIndex):
21                self.traces.append('>'+ruleName)
22
23
24            def traceOut(self, ruleName, ruleIndex):
25                self.traces.append('<'+ruleName)
26
27
28            def recover(self, input, re):
29                # no error recovery yet, just crash!
30                raise
31
32        return TWalker
33
34
35    def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input):
36        lexerCls, parserCls = self.compileInlineGrammar(grammar)
37        walkerCls = self.compileInlineGrammar(treeGrammar)
38
39        cStream = antlr3.StringStream(input)
40        lexer = lexerCls(cStream)
41        tStream = antlr3.CommonTokenStream(lexer)
42        parser = parserCls(tStream)
43        r = getattr(parser, grammarEntry)()
44        nodes = antlr3.tree.CommonTreeNodeStream(r.tree)
45        nodes.setTokenStream(tStream)
46        walker = walkerCls(nodes)
47        getattr(walker, treeEntry)()
48
49        return walker._output
50
51
52    def testFlatList(self):
53        grammar = textwrap.dedent(
54        r'''grammar T;
55        options {
56            language=Python;
57            output=AST;
58        }
59        a : ID INT;
60        ID : 'a'..'z'+ ;
61        INT : '0'..'9'+;
62        WS : (' '|'\n') {$channel=HIDDEN;} ;
63        ''')
64
65        treeGrammar = textwrap.dedent(
66        r'''tree grammar TP;
67        options {
68            language=Python;
69            ASTLabelType=CommonTree;
70        }
71        a : ID INT
72            {self.capture("\%s, \%s" \% ($ID, $INT))}
73          ;
74        ''')
75
76        found = self.execTreeParser(
77            grammar, 'a',
78            treeGrammar, 'a',
79            "abc 34"
80            )
81
82        self.failUnlessEqual("abc, 34", found)
83
84
85
86    def testSimpleTree(self):
87        grammar = textwrap.dedent(
88            r'''grammar T;
89            options {
90                language=Python;
91                output=AST;
92            }
93            a : ID INT -> ^(ID INT);
94            ID : 'a'..'z'+ ;
95            INT : '0'..'9'+;
96            WS : (' '|'\\n') {$channel=HIDDEN;} ;
97            ''')
98
99        treeGrammar = textwrap.dedent(
100            r'''tree grammar TP;
101            options {
102                language=Python;
103                ASTLabelType=CommonTree;
104            }
105            a : ^(ID INT)
106                {self.capture(str($ID)+", "+str($INT))}
107              ;
108            ''')
109
110        found = self.execTreeParser(
111            grammar, 'a',
112            treeGrammar, 'a',
113            "abc 34"
114            )
115
116        self.failUnlessEqual("abc, 34", found)
117
118
119    def testFlatVsTreeDecision(self):
120        grammar = textwrap.dedent(
121            r'''grammar T;
122            options {
123                language=Python;
124                output=AST;
125            }
126            a : b c ;
127            b : ID INT -> ^(ID INT);
128            c : ID INT;
129            ID : 'a'..'z'+ ;
130            INT : '0'..'9'+;
131            WS : (' '|'\\n') {$channel=HIDDEN;} ;
132            ''')
133
134        treeGrammar = textwrap.dedent(
135            r'''tree grammar TP;
136            options {
137                language=Python;
138                ASTLabelType=CommonTree;
139            }
140            a : b b ;
141            b : ID INT    {self.capture(str($ID)+" "+str($INT)+'\n')}
142              | ^(ID INT) {self.capture("^("+str($ID)+" "+str($INT)+')');}
143              ;
144            ''')
145
146        found = self.execTreeParser(
147            grammar, 'a',
148            treeGrammar, 'a',
149            "a 1 b 2"
150            )
151        self.failUnlessEqual("^(a 1)b 2\n", found)
152
153
154    def testFlatVsTreeDecision2(self):
155        grammar = textwrap.dedent(
156            r"""grammar T;
157            options {
158                language=Python;
159                output=AST;
160            }
161            a : b c ;
162            b : ID INT+ -> ^(ID INT+);
163            c : ID INT+;
164            ID : 'a'..'z'+ ;
165            INT : '0'..'9'+;
166            WS : (' '|'\n') {$channel=HIDDEN;} ;
167            """)
168
169        treeGrammar = textwrap.dedent(
170            r'''tree grammar TP;
171            options {
172                language=Python;
173                ASTLabelType=CommonTree;
174            }
175            a : b b ;
176            b : ID INT+    {self.capture(str($ID)+" "+str($INT)+"\n")}
177              | ^(x=ID (y=INT)+) {self.capture("^("+str($x)+' '+str($y)+')')}
178              ;
179            ''')
180
181        found = self.execTreeParser(
182            grammar, 'a',
183            treeGrammar, 'a',
184            "a 1 2 3 b 4 5"
185            )
186        self.failUnlessEqual("^(a 3)b 5\n", found)
187
188
189    def testCyclicDFALookahead(self):
190        grammar = textwrap.dedent(
191            r'''grammar T;
192            options {
193                language=Python;
194                output=AST;
195            }
196            a : ID INT+ PERIOD;
197            ID : 'a'..'z'+ ;
198            INT : '0'..'9'+;
199            SEMI : ';' ;
200            PERIOD : '.' ;
201            WS : (' '|'\n') {$channel=HIDDEN;} ;
202            ''')
203
204        treeGrammar = textwrap.dedent(
205            r'''tree grammar TP;
206            options {
207                language=Python;
208                ASTLabelType=CommonTree;
209            }
210            a : ID INT+ PERIOD {self.capture("alt 1")}
211              | ID INT+ SEMI   {self.capture("alt 2")}
212              ;
213            ''')
214
215        found = self.execTreeParser(
216            grammar, 'a',
217            treeGrammar, 'a',
218            "a 1 2 3."
219            )
220        self.failUnlessEqual("alt 1", found)
221
222
223##     def testTemplateOutput(self):
224## 		String grammar =
225## 			"grammar T;\n" +
226## 			"options {output=AST;}\n" +
227## 			"a : ID INT;\n" +
228## 			"ID : 'a'..'z'+ ;\n" +
229## 			"INT : '0'..'9'+;\n" +
230## 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
231
232## 		String treeGrammar =
233## 			"tree grammar TP;\n" +
234## 			"options {output=template; ASTLabelType=CommonTree;}\n" +
235## 			"s : a {System.out.println($a.st);};\n" +
236## 			"a : ID INT -> {new StringTemplate($INT.text)}\n" +
237## 			"  ;\n";
238
239## 		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
240## 				    treeGrammar, "TP", "TLexer", "a", "s", "abc 34");
241## 		assertEquals("34\n", found);
242## 	}
243
244
245    def testNullableChildList(self):
246        grammar = textwrap.dedent(
247            r'''grammar T;
248            options {
249                language=Python;
250                output=AST;
251            }
252            a : ID INT? -> ^(ID INT?);
253            ID : 'a'..'z'+ ;
254            INT : '0'..'9'+;
255            WS : (' '|'\\n') {$channel=HIDDEN;} ;
256            ''')
257
258        treeGrammar = textwrap.dedent(
259            r'''tree grammar TP;
260            options {
261                language=Python;
262                ASTLabelType=CommonTree;
263            }
264            a : ^(ID INT?)
265                {self.capture(str($ID))}
266              ;
267            ''')
268
269        found = self.execTreeParser(
270            grammar, 'a',
271            treeGrammar, 'a',
272            "abc"
273            )
274        self.failUnlessEqual("abc", found)
275
276
277    def testNullableChildList2(self):
278        grammar = textwrap.dedent(
279            r'''grammar T;
280            options {
281                language=Python;
282                output=AST;
283            }
284            a : ID INT? SEMI -> ^(ID INT?) SEMI ;
285            ID : 'a'..'z'+ ;
286            INT : '0'..'9'+;
287            SEMI : ';' ;
288            WS : (' '|'\n') {$channel=HIDDEN;} ;
289            ''')
290
291        treeGrammar = textwrap.dedent(
292            r'''tree grammar TP;
293            options {
294                language=Python;
295                ASTLabelType=CommonTree;
296            }
297            a : ^(ID INT?) SEMI
298                {self.capture(str($ID))}
299              ;
300            ''')
301
302        found = self.execTreeParser(
303            grammar, 'a',
304            treeGrammar, 'a',
305            "abc;"
306            )
307        self.failUnlessEqual("abc", found)
308
309
310    def testNullableChildList3(self):
311        grammar = textwrap.dedent(
312            r'''grammar T;
313            options {
314                language=Python;
315                output=AST;
316            }
317            a : x=ID INT? (y=ID)? SEMI -> ^($x INT? $y?) SEMI ;
318            ID : 'a'..'z'+ ;
319            INT : '0'..'9'+;
320            SEMI : ';' ;
321            WS : (' '|'\\n') {$channel=HIDDEN;} ;
322            ''')
323
324        treeGrammar = textwrap.dedent(
325            r'''tree grammar TP;
326            options {
327                language=Python;
328                ASTLabelType=CommonTree;
329            }
330            a : ^(ID INT? b) SEMI
331                {self.capture(str($ID)+", "+str($b.text))}
332              ;
333            b : ID? ;
334            ''')
335
336        found = self.execTreeParser(
337            grammar, 'a',
338            treeGrammar, 'a',
339            "abc def;"
340            )
341        self.failUnlessEqual("abc, def", found)
342
343
344    def testActionsAfterRoot(self):
345        grammar = textwrap.dedent(
346            r'''grammar T;
347            options {
348                language=Python;
349                output=AST;
350            }
351            a : x=ID INT? SEMI -> ^($x INT?) ;
352            ID : 'a'..'z'+ ;
353            INT : '0'..'9'+;
354            SEMI : ';' ;
355            WS : (' '|'\n') {$channel=HIDDEN;} ;
356            ''')
357
358        treeGrammar = textwrap.dedent(
359            r'''tree grammar TP;
360            options {
361                language=Python;
362                ASTLabelType=CommonTree;
363            }
364            a @init {x=0} : ^(ID {x=1} {x=2} INT?)
365                {self.capture(str($ID)+", "+str(x))}
366              ;
367            ''')
368
369        found = self.execTreeParser(
370            grammar, 'a',
371            treeGrammar, 'a',
372            "abc;"
373            )
374        self.failUnless("abc, 2\n", found)
375
376
377    def testWildcardLookahead(self):
378        grammar = textwrap.dedent(
379            r'''
380            grammar T;
381            options {language=Python; output=AST;}
382            a : ID '+'^ INT;
383            ID : 'a'..'z'+ ;
384            INT : '0'..'9'+;
385            SEMI : ';' ;
386            PERIOD : '.' ;
387            WS : (' '|'\n') {$channel=HIDDEN;} ;
388            ''')
389
390        treeGrammar = textwrap.dedent(
391            r'''
392            tree grammar TP;
393            options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
394            a : ^('+' . INT) { self.capture("alt 1") }
395              ;
396            ''')
397
398        found = self.execTreeParser(
399            grammar, 'a',
400            treeGrammar, 'a',
401            "a + 2")
402        self.assertEquals("alt 1", found)
403
404
405    def testWildcardLookahead2(self):
406        grammar = textwrap.dedent(
407            r'''
408            grammar T;
409            options {language=Python; output=AST;}
410            a : ID '+'^ INT;
411            ID : 'a'..'z'+ ;
412            INT : '0'..'9'+;
413            SEMI : ';' ;
414            PERIOD : '.' ;
415            WS : (' '|'\n') {$channel=HIDDEN;} ;
416            ''')
417
418        treeGrammar = textwrap.dedent(
419            r'''
420            tree grammar TP;
421            options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
422            a : ^('+' . INT) { self.capture("alt 1") }
423              | ^('+' . .)   { self.capture("alt 2") }
424              ;
425            ''')
426
427        # AMBIG upon '+' DOWN INT UP etc.. but so what.
428
429        found = self.execTreeParser(
430            grammar, 'a',
431            treeGrammar, 'a',
432            "a + 2")
433        self.assertEquals("alt 1", found)
434
435
436    def testWildcardLookahead3(self):
437        grammar = textwrap.dedent(
438            r'''
439            grammar T;
440            options {language=Python; output=AST;}
441            a : ID '+'^ INT;
442            ID : 'a'..'z'+ ;
443            INT : '0'..'9'+;
444            SEMI : ';' ;
445            PERIOD : '.' ;
446            WS : (' '|'\n') {$channel=HIDDEN;} ;
447            ''')
448
449        treeGrammar = textwrap.dedent(
450            r'''
451            tree grammar TP;
452            options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
453            a : ^('+' ID INT) { self.capture("alt 1") }
454              | ^('+' . .)   { self.capture("alt 2") }
455              ;
456            ''')
457
458        # AMBIG upon '+' DOWN INT UP etc.. but so what.
459
460        found = self.execTreeParser(
461            grammar, 'a',
462            treeGrammar, 'a',
463            "a + 2")
464        self.assertEquals("alt 1", found)
465
466
467    def testWildcardPlusLookahead(self):
468        grammar = textwrap.dedent(
469            r'''
470            grammar T;
471            options {language=Python; output=AST;}
472            a : ID '+'^ INT;
473            ID : 'a'..'z'+ ;
474            INT : '0'..'9'+;
475            SEMI : ';' ;
476            PERIOD : '.' ;
477            WS : (' '|'\n') {$channel=HIDDEN;} ;
478            ''')
479
480        treeGrammar = textwrap.dedent(
481            r'''
482            tree grammar TP;
483            options {language=Python; tokenVocab=T; ASTLabelType=CommonTree;}
484            a : ^('+' INT INT ) { self.capture("alt 1") }
485              | ^('+' .+)   { self.capture("alt 2") }
486              ;
487            ''')
488
489        # AMBIG upon '+' DOWN INT UP etc.. but so what.
490
491        found = self.execTreeParser(
492            grammar, 'a',
493            treeGrammar, 'a',
494            "a + 2")
495        self.assertEquals("alt 2", found)
496
497
498if __name__ == '__main__':
499    unittest.main()
500