• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import unittest
2import textwrap
3import antlr3
4import antlr3.tree
5import stringtemplate3
6import testbase
7import sys
8import os
9from StringIO import StringIO
10
11class T(testbase.ANTLRTest):
12    def execParser(self, grammar, grammarEntry, input, group=None):
13        lexerCls, parserCls = self.compileInlineGrammar(grammar)
14
15        cStream = antlr3.StringStream(input)
16        lexer = lexerCls(cStream)
17        tStream = antlr3.CommonTokenStream(lexer)
18        parser = parserCls(tStream)
19        if group is not None:
20            parser.templateLib = group
21        result = getattr(parser, grammarEntry)()
22        if result.st is not None:
23            return result.st.toString()
24        return None
25
26
27    def testInlineTemplate(self):
28        grammar = textwrap.dedent(
29            r'''grammar T;
30            options {
31              language=Python;
32              output=template;
33            }
34            a : ID INT
35              -> template(id={$ID.text}, int={$INT.text})
36                 "id=<id>, int=<int>"
37            ;
38
39            ID : 'a'..'z'+;
40            INT : '0'..'9'+;
41            WS : (' '|'\n') {$channel=HIDDEN;} ;
42            '''
43            )
44
45        found = self.execParser(
46            grammar, 'a',
47            "abc 34"
48            )
49
50        self.failUnlessEqual("id=abc, int=34", found)
51
52
53    def testExternalTemplate(self):
54        templates = textwrap.dedent(
55            '''\
56            group T;
57            expr(args, op) ::= <<
58            [<args; separator={<op>}>]
59            >>
60            '''
61            )
62
63        group = stringtemplate3.StringTemplateGroup(
64            file=StringIO(templates),
65            lexer='angle-bracket'
66            )
67
68        grammar = textwrap.dedent(
69            r'''grammar T2;
70            options {
71              language=Python;
72              output=template;
73            }
74            a : r+=arg OP r+=arg
75              -> expr(op={$OP.text}, args={$r})
76            ;
77            arg: ID -> template(t={$ID.text}) "<t>";
78
79            ID : 'a'..'z'+;
80            OP: '+';
81            WS : (' '|'\n') {$channel=HIDDEN;} ;
82            '''
83            )
84
85        found = self.execParser(
86            grammar, 'a',
87            "a + b",
88            group
89            )
90
91        self.failUnlessEqual("[a+b]", found)
92
93
94    def testEmptyTemplate(self):
95        grammar = textwrap.dedent(
96            r'''grammar T;
97            options {
98              language=Python;
99              output=template;
100            }
101            a : ID INT
102              ->
103            ;
104
105            ID : 'a'..'z'+;
106            INT : '0'..'9'+;
107            WS : (' '|'\n') {$channel=HIDDEN;} ;
108            '''
109            )
110
111        found = self.execParser(
112            grammar, 'a',
113            "abc 34"
114            )
115
116        self.failUnless(found is None)
117
118
119    def testList(self):
120        grammar = textwrap.dedent(
121            r'''grammar T;
122            options {
123              language=Python;
124              output=template;
125            }
126            a: (r+=b)* EOF
127              -> template(r={$r})
128                 "<r; separator=\",\">"
129            ;
130
131            b: ID
132              -> template(t={$ID.text}) "<t>"
133            ;
134
135            ID : 'a'..'z'+;
136            WS : (' '|'\n') {$channel=HIDDEN;} ;
137            '''
138            )
139
140        found = self.execParser(
141            grammar, 'a',
142            "abc def ghi"
143            )
144
145        self.failUnlessEqual("abc,def,ghi", found)
146
147
148    def testAction(self):
149        grammar = textwrap.dedent(
150            r'''grammar T;
151            options {
152              language=Python;
153              output=template;
154            }
155            a: ID
156              -> { stringtemplate3.StringTemplate("hello") }
157            ;
158
159            ID : 'a'..'z'+;
160            WS : (' '|'\n') {$channel=HIDDEN;} ;
161            '''
162            )
163
164        found = self.execParser(
165            grammar, 'a',
166            "abc"
167            )
168
169        self.failUnlessEqual("hello", found)
170
171
172    def testTemplateExpressionInAction(self):
173        grammar = textwrap.dedent(
174            r'''grammar T;
175            options {
176              language=Python;
177              output=template;
178            }
179            a: ID
180              { $st = %{"hello"} }
181            ;
182
183            ID : 'a'..'z'+;
184            WS : (' '|'\n') {$channel=HIDDEN;} ;
185            '''
186            )
187
188        found = self.execParser(
189            grammar, 'a',
190            "abc"
191            )
192
193        self.failUnlessEqual("hello", found)
194
195
196    def testTemplateExpressionInAction2(self):
197        grammar = textwrap.dedent(
198            r'''grammar T;
199            options {
200              language=Python;
201              output=template;
202            }
203            a: ID
204              {
205                res = %{"hello <foo>"}
206                %res.foo = "world";
207              }
208              -> { res }
209            ;
210
211            ID : 'a'..'z'+;
212            WS : (' '|'\n') {$channel=HIDDEN;} ;
213            '''
214            )
215
216        found = self.execParser(
217            grammar, 'a',
218            "abc"
219            )
220
221        self.failUnlessEqual("hello world", found)
222
223
224    def testIndirectTemplateConstructor(self):
225        templates = textwrap.dedent(
226            '''\
227            group T;
228            expr(args, op) ::= <<
229            [<args; separator={<op>}>]
230            >>
231            '''
232            )
233
234        group = stringtemplate3.StringTemplateGroup(
235            file=StringIO(templates),
236            lexer='angle-bracket'
237            )
238
239        grammar = textwrap.dedent(
240            r'''grammar T;
241            options {
242              language=Python;
243              output=template;
244            }
245            a: ID
246              {
247                $st = %({"expr"})(args={[1, 2, 3]}, op={"+"})
248              }
249            ;
250
251            ID : 'a'..'z'+;
252            WS : (' '|'\n') {$channel=HIDDEN;} ;
253            '''
254            )
255
256        found = self.execParser(
257            grammar, 'a',
258            "abc",
259            group
260            )
261
262        self.failUnlessEqual("[1+2+3]", found)
263
264
265    def testPredicates(self):
266        grammar = textwrap.dedent(
267            r'''grammar T3;
268            options {
269              language=Python;
270              output=template;
271            }
272            a : ID INT
273              -> {$ID.text=='a'}? template(int={$INT.text})
274                                  "A: <int>"
275              -> {$ID.text=='b'}? template(int={$INT.text})
276                                  "B: <int>"
277              ->                  template(int={$INT.text})
278                                  "C: <int>"
279            ;
280
281            ID : 'a'..'z'+;
282            INT : '0'..'9'+;
283            WS : (' '|'\n') {$channel=HIDDEN;} ;
284            '''
285            )
286
287        found = self.execParser(
288            grammar, 'a',
289            "b 34"
290            )
291
292        self.failUnlessEqual("B: 34", found)
293
294
295    def testBacktrackingMode(self):
296        grammar = textwrap.dedent(
297            r'''grammar T4;
298            options {
299              language=Python;
300              output=template;
301              backtrack=true;
302            }
303            a : (ID INT)=> ID INT
304              -> template(id={$ID.text}, int={$INT.text})
305                 "id=<id>, int=<int>"
306            ;
307
308            ID : 'a'..'z'+;
309            INT : '0'..'9'+;
310            WS : (' '|'\n') {$channel=HIDDEN;} ;
311            '''
312            )
313
314        found = self.execParser(
315            grammar, 'a',
316            "abc 34"
317            )
318
319        self.failUnlessEqual("id=abc, int=34", found)
320
321
322    def testRewrite(self):
323        grammar = textwrap.dedent(
324            r'''grammar T5;
325            options {
326              language=Python;
327              output=template;
328              rewrite=true;
329            }
330
331            prog: stat+;
332
333            stat
334                : 'if' '(' expr ')' stat
335                | 'return' return_expr ';'
336                | '{' stat* '}'
337                | ID '=' expr ';'
338                ;
339
340            return_expr
341                : expr
342                  -> template(t={$text}) <<boom(<t>)>>
343                ;
344
345            expr
346                : ID
347                | INT
348                ;
349
350            ID:  'a'..'z'+;
351            INT: '0'..'9'+;
352            WS: (' '|'\n')+ {$channel=HIDDEN;} ;
353            COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ;
354            '''
355            )
356
357        input = textwrap.dedent(
358            '''\
359            if ( foo ) {
360              b = /* bla */ 2;
361              return 1 /* foo */;
362            }
363
364            /* gnurz */
365            return 12;
366            '''
367            )
368
369        lexerCls, parserCls = self.compileInlineGrammar(grammar)
370
371        cStream = antlr3.StringStream(input)
372        lexer = lexerCls(cStream)
373        tStream = antlr3.TokenRewriteStream(lexer)
374        parser = parserCls(tStream)
375        result = parser.prog()
376
377        found = tStream.toString()
378
379        expected = textwrap.dedent(
380            '''\
381            if ( foo ) {
382              b = /* bla */ 2;
383              return boom(1) /* foo */;
384            }
385
386            /* gnurz */
387            return boom(12);
388            '''
389            )
390
391        self.failUnlessEqual(expected, found)
392
393
394    def testTreeRewrite(self):
395        grammar = textwrap.dedent(
396            r'''grammar T6;
397            options {
398              language=Python;
399              output=AST;
400            }
401
402            tokens {
403              BLOCK;
404              ASSIGN;
405            }
406
407            prog: stat+;
408
409            stat
410                : IF '(' e=expr ')' s=stat
411                  -> ^(IF $e $s)
412                | RETURN expr ';'
413                  -> ^(RETURN expr)
414                | '{' stat* '}'
415                  -> ^(BLOCK stat*)
416                | ID '=' expr ';'
417                  -> ^(ASSIGN ID expr)
418                ;
419
420            expr
421                : ID
422                | INT
423                ;
424
425            IF: 'if';
426            RETURN: 'return';
427            ID:  'a'..'z'+;
428            INT: '0'..'9'+;
429            WS: (' '|'\n')+ {$channel=HIDDEN;} ;
430            COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ;
431            '''
432            )
433
434        treeGrammar = textwrap.dedent(
435            r'''tree grammar T6Walker;
436            options {
437              language=Python;
438              tokenVocab=T6;
439              ASTLabelType=CommonTree;
440              output=template;
441              rewrite=true;
442            }
443
444            prog: stat+;
445
446            stat
447                : ^(IF expr stat)
448                | ^(RETURN return_expr)
449                | ^(BLOCK stat*)
450                | ^(ASSIGN ID expr)
451                ;
452
453            return_expr
454                : expr
455                  -> template(t={$text}) <<boom(<t>)>>
456                ;
457
458            expr
459                : ID
460                | INT
461                ;
462            '''
463            )
464
465        input = textwrap.dedent(
466            '''\
467            if ( foo ) {
468              b = /* bla */ 2;
469              return 1 /* foo */;
470            }
471
472            /* gnurz */
473            return 12;
474            '''
475            )
476
477        lexerCls, parserCls = self.compileInlineGrammar(grammar)
478        walkerCls = self.compileInlineGrammar(treeGrammar)
479
480        cStream = antlr3.StringStream(input)
481        lexer = lexerCls(cStream)
482        tStream = antlr3.TokenRewriteStream(lexer)
483        parser = parserCls(tStream)
484        tree = parser.prog().tree
485        nodes = antlr3.tree.CommonTreeNodeStream(tree)
486        nodes.setTokenStream(tStream)
487        walker = walkerCls(nodes)
488        walker.prog()
489
490        found = tStream.toString()
491
492        expected = textwrap.dedent(
493            '''\
494            if ( foo ) {
495              b = /* bla */ 2;
496              return boom(1) /* foo */;
497            }
498
499            /* gnurz */
500            return boom(12);
501            '''
502            )
503
504        self.failUnlessEqual(expected, found)
505
506
507if __name__ == '__main__':
508    unittest.main()
509