1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 import org.antlr.Tool; 31 import org.antlr.tool.*; 32 import org.junit.Test; 33 34 import java.io.File; 35 36 public class TestCompositeGrammars extends BaseTest { 37 protected boolean debug = false; 38 testWildcardStillWorks()39 @Test public void testWildcardStillWorks() throws Exception { 40 ErrorQueue equeue = new ErrorQueue(); 41 ErrorManager.setErrorListener(equeue); 42 String grammar = 43 "parser grammar S;\n" + 44 "a : B . C ;\n"; // not qualified ID 45 Grammar g = new Grammar(grammar); 46 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 47 } 48 testDelegatorInvokesDelegateRule()49 @Test public void testDelegatorInvokesDelegateRule() throws Exception { 50 String slave = 51 "parser grammar S;\n" + 52 "a : B {System.out.println(\"S.a\");} ;\n"; 53 mkdir(tmpdir); 54 writeFile(tmpdir, "S.g", slave); 55 String master = 56 "grammar M;\n" + 57 "import S;\n" + 58 "s : a ;\n" + 59 "B : 'b' ;" + // defines B from inherited token space 60 "WS : (' '|'\\n') {skip();} ;\n" ; 61 String found = execParser("M.g", master, "MParser", "MLexer", 62 "s", "b", debug); 63 assertEquals("S.a\n", found); 64 } 65 testDelegatorInvokesDelegateRuleWithArgs()66 @Test public void testDelegatorInvokesDelegateRuleWithArgs() throws Exception { 67 // must generate something like: 68 // public int a(int x) throws RecognitionException { return gS.a(x); } 69 // in M. 70 String slave = 71 "parser grammar S;\n" + 72 "a[int x] returns [int y] : B {System.out.print(\"S.a\"); $y=1000;} ;\n"; 73 mkdir(tmpdir); 74 writeFile(tmpdir, "S.g", slave); 75 String master = 76 "grammar M;\n" + 77 "import S;\n" + 78 "s : label=a[3] {System.out.println($label.y);} ;\n" + 79 "B : 'b' ;" + // defines B from inherited token space 80 "WS : (' '|'\\n') {skip();} ;\n" ; 81 String found = execParser("M.g", master, "MParser", "MLexer", 82 "s", "b", debug); 83 assertEquals("S.a1000\n", found); 84 } 85 testDelegatorInvokesDelegateRuleWithReturnStruct()86 @Test public void testDelegatorInvokesDelegateRuleWithReturnStruct() throws Exception { 87 // must generate something like: 88 // public int a(int x) throws RecognitionException { return gS.a(x); } 89 // in M. 90 String slave = 91 "parser grammar S;\n" + 92 "a : B {System.out.print(\"S.a\");} ;\n"; 93 mkdir(tmpdir); 94 writeFile(tmpdir, "S.g", slave); 95 String master = 96 "grammar M;\n" + 97 "import S;\n" + 98 "s : a {System.out.println($a.text);} ;\n" + 99 "B : 'b' ;" + // defines B from inherited token space 100 "WS : (' '|'\\n') {skip();} ;\n" ; 101 String found = execParser("M.g", master, "MParser", "MLexer", 102 "s", "b", debug); 103 assertEquals("S.ab\n", found); 104 } 105 testDelegatorAccessesDelegateMembers()106 @Test public void testDelegatorAccessesDelegateMembers() throws Exception { 107 String slave = 108 "parser grammar S;\n" + 109 "@members {\n" + 110 " public void foo() {System.out.println(\"foo\");}\n" + 111 "}\n" + 112 "a : B ;\n"; 113 mkdir(tmpdir); 114 writeFile(tmpdir, "S.g", slave); 115 String master = 116 "grammar M;\n" + // uses no rules from the import 117 "import S;\n" + 118 "s : 'b' {gS.foo();} ;\n" + // gS is import pointer 119 "WS : (' '|'\\n') {skip();} ;\n" ; 120 String found = execParser("M.g", master, "MParser", "MLexer", 121 "s", "b", debug); 122 assertEquals("foo\n", found); 123 } 124 testDelegatorInvokesFirstVersionOfDelegateRule()125 @Test public void testDelegatorInvokesFirstVersionOfDelegateRule() throws Exception { 126 String slave = 127 "parser grammar S;\n" + 128 "a : b {System.out.println(\"S.a\");} ;\n" + 129 "b : B ;\n" ; 130 mkdir(tmpdir); 131 writeFile(tmpdir, "S.g", slave); 132 String slave2 = 133 "parser grammar T;\n" + 134 "a : B {System.out.println(\"T.a\");} ;\n"; // hidden by S.a 135 writeFile(tmpdir, "T.g", slave2); 136 String master = 137 "grammar M;\n" + 138 "import S,T;\n" + 139 "s : a ;\n" + 140 "B : 'b' ;\n" + 141 "WS : (' '|'\\n') {skip();} ;\n" ; 142 String found = execParser("M.g", master, "MParser", "MLexer", 143 "s", "b", debug); 144 assertEquals("S.a\n", found); 145 } 146 testDelegatesSeeSameTokenType()147 @Test public void testDelegatesSeeSameTokenType() throws Exception { 148 String slave = 149 "parser grammar S;\n" + // A, B, C token type order 150 "tokens { A; B; C; }\n" + 151 "x : A {System.out.println(\"S.x\");} ;\n"; 152 mkdir(tmpdir); 153 writeFile(tmpdir, "S.g", slave); 154 String slave2 = 155 "parser grammar T;\n" + 156 "tokens { C; B; A; }\n" + // reverse order 157 "y : A {System.out.println(\"T.y\");} ;\n"; 158 mkdir(tmpdir); 159 writeFile(tmpdir, "T.g", slave2); 160 // The lexer will create rules to match letters a, b, c. 161 // The associated token types A, B, C must have the same value 162 // and all import'd parsers. Since ANTLR regenerates all imports 163 // for use with the delegator M, it can generate the same token type 164 // mapping in each parser: 165 // public static final int C=6; 166 // public static final int EOF=-1; 167 // public static final int B=5; 168 // public static final int WS=7; 169 // public static final int A=4; 170 171 String master = 172 "grammar M;\n" + 173 "import S,T;\n" + 174 "s : x y ;\n" + // matches AA, which should be "aa" 175 "B : 'b' ;\n" + // another order: B, A, C 176 "A : 'a' ;\n" + 177 "C : 'c' ;\n" + 178 "WS : (' '|'\\n') {skip();} ;\n" ; 179 String found = execParser("M.g", master, "MParser", "MLexer", 180 "s", "aa", debug); 181 assertEquals("S.x\n" + 182 "T.y\n", found); 183 } 184 testDelegatesSeeSameTokenType2()185 @Test public void testDelegatesSeeSameTokenType2() throws Exception { 186 ErrorQueue equeue = new ErrorQueue(); 187 ErrorManager.setErrorListener(equeue); 188 String slave = 189 "parser grammar S;\n" + // A, B, C token type order 190 "tokens { A; B; C; }\n" + 191 "x : A {System.out.println(\"S.x\");} ;\n"; 192 mkdir(tmpdir); 193 writeFile(tmpdir, "S.g", slave); 194 String slave2 = 195 "parser grammar T;\n" + 196 "tokens { C; B; A; }\n" + // reverse order 197 "y : A {System.out.println(\"T.y\");} ;\n"; 198 mkdir(tmpdir); 199 writeFile(tmpdir, "T.g", slave2); 200 201 String master = 202 "grammar M;\n" + 203 "import S,T;\n" + 204 "s : x y ;\n" + // matches AA, which should be "aa" 205 "B : 'b' ;\n" + // another order: B, A, C 206 "A : 'a' ;\n" + 207 "C : 'c' ;\n" + 208 "WS : (' '|'\\n') {skip();} ;\n" ; 209 writeFile(tmpdir, "M.g", master); 210 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 211 CompositeGrammar composite = new CompositeGrammar(); 212 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 213 composite.setDelegationRoot(g); 214 g.parseAndBuildAST(); 215 g.composite.assignTokenTypes(); 216 217 String expectedTokenIDToTypeMap = "[A=4, B=5, C=6, WS=7]"; 218 String expectedStringLiteralToTypeMap = "{}"; 219 String expectedTypeToTokenList = "[A, B, C, WS]"; 220 221 assertEquals(expectedTokenIDToTypeMap, 222 realElements(g.composite.tokenIDToTypeMap).toString()); 223 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 224 assertEquals(expectedTypeToTokenList, 225 realElements(g.composite.typeToTokenList).toString()); 226 227 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 228 } 229 testCombinedImportsCombined()230 @Test public void testCombinedImportsCombined() throws Exception { 231 // for now, we don't allow combined to import combined 232 ErrorQueue equeue = new ErrorQueue(); 233 ErrorManager.setErrorListener(equeue); 234 String slave = 235 "grammar S;\n" + // A, B, C token type order 236 "tokens { A; B; C; }\n" + 237 "x : 'x' INT {System.out.println(\"S.x\");} ;\n" + 238 "INT : '0'..'9'+ ;\n" + 239 "WS : (' '|'\\n') {skip();} ;\n"; 240 mkdir(tmpdir); 241 writeFile(tmpdir, "S.g", slave); 242 243 String master = 244 "grammar M;\n" + 245 "import S;\n" + 246 "s : x INT ;\n"; 247 writeFile(tmpdir, "M.g", master); 248 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 249 CompositeGrammar composite = new CompositeGrammar(); 250 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 251 composite.setDelegationRoot(g); 252 g.parseAndBuildAST(); 253 g.composite.assignTokenTypes(); 254 255 assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); 256 String expectedError = "error(161): "+tmpdir.toString().replaceFirst("\\-[0-9]+","")+"/M.g:2:8: combined grammar M cannot import combined grammar S"; 257 assertEquals("unexpected errors: "+equeue, expectedError, equeue.errors.get(0).toString().replaceFirst("\\-[0-9]+","")); 258 } 259 testSameStringTwoNames()260 @Test public void testSameStringTwoNames() throws Exception { 261 ErrorQueue equeue = new ErrorQueue(); 262 ErrorManager.setErrorListener(equeue); 263 String slave = 264 "parser grammar S;\n" + 265 "tokens { A='a'; }\n" + 266 "x : A {System.out.println(\"S.x\");} ;\n"; 267 mkdir(tmpdir); 268 writeFile(tmpdir, "S.g", slave); 269 String slave2 = 270 "parser grammar T;\n" + 271 "tokens { X='a'; }\n" + 272 "y : X {System.out.println(\"T.y\");} ;\n"; 273 mkdir(tmpdir); 274 writeFile(tmpdir, "T.g", slave2); 275 276 String master = 277 "grammar M;\n" + 278 "import S,T;\n" + 279 "s : x y ;\n" + 280 "WS : (' '|'\\n') {skip();} ;\n" ; 281 writeFile(tmpdir, "M.g", master); 282 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 283 CompositeGrammar composite = new CompositeGrammar(); 284 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 285 composite.setDelegationRoot(g); 286 g.parseAndBuildAST(); 287 g.composite.assignTokenTypes(); 288 289 String expectedTokenIDToTypeMap = "[A=4, WS=5, X=6]"; 290 String expectedStringLiteralToTypeMap = "{'a'=4}"; 291 String expectedTypeToTokenList = "[A, WS, X]"; 292 293 assertEquals(expectedTokenIDToTypeMap, 294 realElements(g.composite.tokenIDToTypeMap).toString()); 295 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 296 assertEquals(expectedTypeToTokenList, 297 realElements(g.composite.typeToTokenList).toString()); 298 299 Object expectedArg = "X='a'"; 300 Object expectedArg2 = "A"; 301 int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_CONFLICT; 302 GrammarSemanticsMessage expectedMessage = 303 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 304 checkGrammarSemanticsError(equeue, expectedMessage); 305 306 assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); 307 308 String expectedError = 309 "error(158): T.g:2:10: cannot alias X='a'; string already assigned to A"; 310 assertEquals(expectedError, equeue.errors.get(0).toString()); 311 } 312 testSameNameTwoStrings()313 @Test public void testSameNameTwoStrings() throws Exception { 314 ErrorQueue equeue = new ErrorQueue(); 315 ErrorManager.setErrorListener(equeue); 316 String slave = 317 "parser grammar S;\n" + 318 "tokens { A='a'; }\n" + 319 "x : A {System.out.println(\"S.x\");} ;\n"; 320 mkdir(tmpdir); 321 writeFile(tmpdir, "S.g", slave); 322 String slave2 = 323 "parser grammar T;\n" + 324 "tokens { A='x'; }\n" + 325 "y : A {System.out.println(\"T.y\");} ;\n"; 326 327 writeFile(tmpdir, "T.g", slave2); 328 329 String master = 330 "grammar M;\n" + 331 "import S,T;\n" + 332 "s : x y ;\n" + 333 "WS : (' '|'\\n') {skip();} ;\n" ; 334 writeFile(tmpdir, "M.g", master); 335 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 336 CompositeGrammar composite = new CompositeGrammar(); 337 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 338 composite.setDelegationRoot(g); 339 g.parseAndBuildAST(); 340 g.composite.assignTokenTypes(); 341 342 String expectedTokenIDToTypeMap = "[A=4, T__6=6, WS=5]"; 343 String expectedStringLiteralToTypeMap = "{'a'=4, 'x'=6}"; 344 String expectedTypeToTokenList = "[A, WS, T__6]"; 345 346 assertEquals(expectedTokenIDToTypeMap, 347 realElements(g.composite.tokenIDToTypeMap).toString()); 348 assertEquals(expectedStringLiteralToTypeMap, sortMapToString(g.composite.stringLiteralToTypeMap)); 349 assertEquals(expectedTypeToTokenList, 350 realElements(g.composite.typeToTokenList).toString()); 351 352 Object expectedArg = "A='x'"; 353 Object expectedArg2 = "'a'"; 354 int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT; 355 GrammarSemanticsMessage expectedMessage = 356 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 357 checkGrammarSemanticsError(equeue, expectedMessage); 358 359 assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); 360 361 String expectedError = 362 "error(159): T.g:2:10: cannot alias A='x'; token name already assigned to 'a'"; 363 assertEquals(expectedError, equeue.errors.get(0).toString()); 364 } 365 testImportedTokenVocabIgnoredWithWarning()366 @Test public void testImportedTokenVocabIgnoredWithWarning() throws Exception { 367 ErrorQueue equeue = new ErrorQueue(); 368 ErrorManager.setErrorListener(equeue); 369 String slave = 370 "parser grammar S;\n" + 371 "options {tokenVocab=whatever;}\n" + 372 "tokens { A='a'; }\n" + 373 "x : A {System.out.println(\"S.x\");} ;\n"; 374 mkdir(tmpdir); 375 writeFile(tmpdir, "S.g", slave); 376 377 String master = 378 "grammar M;\n" + 379 "import S;\n" + 380 "s : x ;\n" + 381 "WS : (' '|'\\n') {skip();} ;\n" ; 382 writeFile(tmpdir, "M.g", master); 383 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 384 CompositeGrammar composite = new CompositeGrammar(); 385 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 386 composite.setDelegationRoot(g); 387 g.parseAndBuildAST(); 388 g.composite.assignTokenTypes(); 389 390 Object expectedArg = "S"; 391 int expectedMsgID = ErrorManager.MSG_TOKEN_VOCAB_IN_DELEGATE; 392 GrammarSemanticsMessage expectedMessage = 393 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg); 394 checkGrammarSemanticsWarning(equeue, expectedMessage); 395 396 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 397 assertEquals("unexpected errors: "+equeue, 1, equeue.warnings.size()); 398 399 String expectedError = 400 "warning(160): S.g:2:10: tokenVocab option ignored in imported grammar S"; 401 assertEquals(expectedError, equeue.warnings.get(0).toString()); 402 } 403 testImportedTokenVocabWorksInRoot()404 @Test public void testImportedTokenVocabWorksInRoot() throws Exception { 405 ErrorQueue equeue = new ErrorQueue(); 406 ErrorManager.setErrorListener(equeue); 407 String slave = 408 "parser grammar S;\n" + 409 "tokens { A='a'; }\n" + 410 "x : A {System.out.println(\"S.x\");} ;\n"; 411 mkdir(tmpdir); 412 writeFile(tmpdir, "S.g", slave); 413 414 String tokens = 415 "A=99\n"; 416 writeFile(tmpdir, "Test.tokens", tokens); 417 418 String master = 419 "grammar M;\n" + 420 "options {tokenVocab=Test;}\n" + 421 "import S;\n" + 422 "s : x ;\n" + 423 "WS : (' '|'\\n') {skip();} ;\n" ; 424 writeFile(tmpdir, "M.g", master); 425 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 426 CompositeGrammar composite = new CompositeGrammar(); 427 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 428 composite.setDelegationRoot(g); 429 g.parseAndBuildAST(); 430 g.composite.assignTokenTypes(); 431 432 String expectedTokenIDToTypeMap = "[A=99, WS=101]"; 433 String expectedStringLiteralToTypeMap = "{'a'=100}"; 434 String expectedTypeToTokenList = "[A, 'a', WS]"; 435 436 assertEquals(expectedTokenIDToTypeMap, 437 realElements(g.composite.tokenIDToTypeMap).toString()); 438 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 439 assertEquals(expectedTypeToTokenList, 440 realElements(g.composite.typeToTokenList).toString()); 441 442 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 443 } 444 testSyntaxErrorsInImportsNotThrownOut()445 @Test public void testSyntaxErrorsInImportsNotThrownOut() throws Exception { 446 ErrorQueue equeue = new ErrorQueue(); 447 ErrorManager.setErrorListener(equeue); 448 String slave = 449 "parser grammar S;\n" + 450 "options {toke\n"; 451 mkdir(tmpdir); 452 writeFile(tmpdir, "S.g", slave); 453 454 String master = 455 "grammar M;\n" + 456 "import S;\n" + 457 "s : x ;\n" + 458 "WS : (' '|'\\n') {skip();} ;\n" ; 459 writeFile(tmpdir, "M.g", master); 460 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 461 CompositeGrammar composite = new CompositeGrammar(); 462 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 463 composite.setDelegationRoot(g); 464 g.parseAndBuildAST(); 465 g.composite.assignTokenTypes(); 466 467 // whole bunch of errors from bad S.g file 468 assertEquals("unexpected errors: "+equeue, 5, equeue.errors.size()); 469 } 470 testSyntaxErrorsInImportsNotThrownOut2()471 @Test public void testSyntaxErrorsInImportsNotThrownOut2() throws Exception { 472 ErrorQueue equeue = new ErrorQueue(); 473 ErrorManager.setErrorListener(equeue); 474 String slave = 475 "parser grammar S;\n" + 476 ": A {System.out.println(\"S.x\");} ;\n"; 477 mkdir(tmpdir); 478 writeFile(tmpdir, "S.g", slave); 479 480 String master = 481 "grammar M;\n" + 482 "import S;\n" + 483 "s : x ;\n" + 484 "WS : (' '|'\\n') {skip();} ;\n" ; 485 writeFile(tmpdir, "M.g", master); 486 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 487 CompositeGrammar composite = new CompositeGrammar(); 488 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 489 composite.setDelegationRoot(g); 490 g.parseAndBuildAST(); 491 g.composite.assignTokenTypes(); 492 493 // whole bunch of errors from bad S.g file 494 assertEquals("unexpected errors: "+equeue, 3, equeue.errors.size()); 495 } 496 testDelegatorRuleOverridesDelegate()497 @Test public void testDelegatorRuleOverridesDelegate() throws Exception { 498 String slave = 499 "parser grammar S;\n" + 500 "a : b {System.out.println(\"S.a\");} ;\n" + 501 "b : B ;\n" ; 502 mkdir(tmpdir); 503 writeFile(tmpdir, "S.g", slave); 504 String master = 505 "grammar M;\n" + 506 "import S;\n" + 507 "b : 'b'|'c' ;\n" + 508 "WS : (' '|'\\n') {skip();} ;\n" ; 509 String found = execParser("M.g", master, "MParser", "MLexer", 510 "a", "c", debug); 511 assertEquals("S.a\n", found); 512 } 513 testDelegatorRuleOverridesLookaheadInDelegate()514 @Test public void testDelegatorRuleOverridesLookaheadInDelegate() throws Exception { 515 String slave = 516 "parser grammar JavaDecl;\n" + 517 "type : 'int' ;\n" + 518 "decl : type ID ';'\n" + 519 " | type ID init ';' {System.out.println(\"JavaDecl: \"+$decl.text);}\n" + 520 " ;\n" + 521 "init : '=' INT ;\n" ; 522 mkdir(tmpdir); 523 writeFile(tmpdir, "JavaDecl.g", slave); 524 String master = 525 "grammar Java;\n" + 526 "import JavaDecl;\n" + 527 "prog : decl ;\n" + 528 "type : 'int' | 'float' ;\n" + 529 "\n" + 530 "ID : 'a'..'z'+ ;\n" + 531 "INT : '0'..'9'+ ;\n" + 532 "WS : (' '|'\\n') {skip();} ;\n" ; 533 // for float to work in decl, type must be overridden 534 String found = execParser("Java.g", master, "JavaParser", "JavaLexer", 535 "prog", "float x = 3;", debug); 536 assertEquals("JavaDecl: floatx=3;\n", found); 537 } 538 testDelegatorRuleOverridesDelegates()539 @Test public void testDelegatorRuleOverridesDelegates() throws Exception { 540 String slave = 541 "parser grammar S;\n" + 542 "a : b {System.out.println(\"S.a\");} ;\n" + 543 "b : B ;\n" ; 544 mkdir(tmpdir); 545 writeFile(tmpdir, "S.g", slave); 546 547 String slave2 = 548 "parser grammar T;\n" + 549 "tokens { A='x'; }\n" + 550 "b : B {System.out.println(\"T.b\");} ;\n"; 551 writeFile(tmpdir, "T.g", slave2); 552 553 String master = 554 "grammar M;\n" + 555 "import S, T;\n" + 556 "b : 'b'|'c' {System.out.println(\"M.b\");}|B|A ;\n" + 557 "WS : (' '|'\\n') {skip();} ;\n" ; 558 String found = execParser("M.g", master, "MParser", "MLexer", 559 "a", "c", debug); 560 assertEquals("M.b\n" + 561 "S.a\n", found); 562 } 563 564 // LEXER INHERITANCE 565 testLexerDelegatorInvokesDelegateRule()566 @Test public void testLexerDelegatorInvokesDelegateRule() throws Exception { 567 String slave = 568 "lexer grammar S;\n" + 569 "A : 'a' {System.out.println(\"S.A\");} ;\n" + 570 "C : 'c' ;\n"; 571 mkdir(tmpdir); 572 writeFile(tmpdir, "S.g", slave); 573 String master = 574 "lexer grammar M;\n" + 575 "import S;\n" + 576 "B : 'b' ;\n" + 577 "WS : (' '|'\\n') {skip();} ;\n" ; 578 String found = execLexer("M.g", master, "M", "abc", debug); 579 assertEquals("S.A\nabc\n", found); 580 } 581 testLexerDelegatorRuleOverridesDelegate()582 @Test public void testLexerDelegatorRuleOverridesDelegate() throws Exception { 583 String slave = 584 "lexer grammar S;\n" + 585 "A : 'a' {System.out.println(\"S.A\");} ;\n" + 586 "B : 'b' {System.out.println(\"S.B\");} ;\n"; 587 mkdir(tmpdir); 588 writeFile(tmpdir, "S.g", slave); 589 String master = 590 "lexer grammar M;\n" + 591 "import S;\n" + 592 "A : 'a' B {System.out.println(\"M.A\");} ;\n" + 593 "WS : (' '|'\\n') {skip();} ;\n" ; 594 String found = execLexer("M.g", master, "M", "ab", debug); 595 assertEquals("S.B\n" + 596 "M.A\n" + 597 "ab\n", found); 598 } 599 testLexerDelegatorRuleOverridesDelegateLeavingNoRules()600 @Test public void testLexerDelegatorRuleOverridesDelegateLeavingNoRules() throws Exception { 601 // M.Tokens has nothing to predict tokens from S. Should 602 // not include S.Tokens alt in this case? 603 String slave = 604 "lexer grammar S;\n" + 605 "A : 'a' {System.out.println(\"S.A\");} ;\n"; 606 mkdir(tmpdir); 607 writeFile(tmpdir, "S.g", slave); 608 String master = 609 "lexer grammar M;\n" + 610 "import S;\n" + 611 "A : 'a' {System.out.println(\"M.A\");} ;\n" + 612 "WS : (' '|'\\n') {skip();} ;\n" ; 613 writeFile(tmpdir, "/M.g", master); 614 615 ErrorQueue equeue = new ErrorQueue(); 616 ErrorManager.setErrorListener(equeue); 617 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 618 CompositeGrammar composite = new CompositeGrammar(); 619 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 620 composite.setDelegationRoot(g); 621 g.parseAndBuildAST(); 622 composite.assignTokenTypes(); 623 composite.defineGrammarSymbols(); 624 composite.createNFAs(); 625 g.createLookaheadDFAs(false); 626 627 // predict only alts from M not S 628 String expectingDFA = 629 ".s0-'a'->.s1\n" + 630 ".s0-{'\\n', ' '}->:s3=>2\n" + 631 ".s1-<EOT>->:s2=>1\n"; 632 org.antlr.analysis.DFA dfa = g.getLookaheadDFA(1); 633 FASerializer serializer = new FASerializer(g); 634 String result = serializer.serialize(dfa.startState); 635 assertEquals(expectingDFA, result); 636 637 // must not be a "unreachable alt: Tokens" error 638 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 639 } 640 testInvalidImportMechanism()641 @Test public void testInvalidImportMechanism() throws Exception { 642 // M.Tokens has nothing to predict tokens from S. Should 643 // not include S.Tokens alt in this case? 644 String slave = 645 "lexer grammar S;\n" + 646 "A : 'a' {System.out.println(\"S.A\");} ;\n"; 647 mkdir(tmpdir); 648 writeFile(tmpdir, "S.g", slave); 649 String master = 650 "tree grammar M;\n" + 651 "import S;\n" + 652 "a : A ;"; 653 writeFile(tmpdir, "/M.g", master); 654 655 ErrorQueue equeue = new ErrorQueue(); 656 ErrorManager.setErrorListener(equeue); 657 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 658 CompositeGrammar composite = new CompositeGrammar(); 659 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 660 composite.setDelegationRoot(g); 661 g.parseAndBuildAST(); 662 663 assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); 664 assertEquals("unexpected errors: "+equeue, 0, equeue.warnings.size()); 665 666 String expectedError = 667 "error(161): "+tmpdir.toString().replaceFirst("\\-[0-9]+","")+"/M.g:2:8: tree grammar M cannot import lexer grammar S"; 668 assertEquals(expectedError, equeue.errors.get(0).toString().replaceFirst("\\-[0-9]+","")); 669 } 670 testSyntacticPredicateRulesAreNotInherited()671 @Test public void testSyntacticPredicateRulesAreNotInherited() throws Exception { 672 // if this compiles, it means that synpred1_S is defined in S.java 673 // but not MParser.java. MParser has its own synpred1_M which must 674 // be separate to compile. 675 String slave = 676 "parser grammar S;\n" + 677 "a : 'a' {System.out.println(\"S.a1\");}\n" + 678 " | 'a' {System.out.println(\"S.a2\");}\n" + 679 " ;\n" + 680 "b : 'x' | 'y' {;} ;\n"; // preds generated but not need in DFA here 681 mkdir(tmpdir); 682 writeFile(tmpdir, "S.g", slave); 683 String master = 684 "grammar M;\n" + 685 "options {backtrack=true;}\n" + 686 "import S;\n" + 687 "start : a b ;\n" + 688 "nonsense : 'q' | 'q' {;} ;" + // forces def of preds here in M 689 "WS : (' '|'\\n') {skip();} ;\n" ; 690 String found = execParser("M.g", master, "MParser", "MLexer", 691 "start", "ax", debug); 692 assertEquals("S.a1\n", found); 693 } 694 testKeywordVSIDGivesNoWarning()695 @Test public void testKeywordVSIDGivesNoWarning() throws Exception { 696 ErrorQueue equeue = new ErrorQueue(); 697 ErrorManager.setErrorListener(equeue); 698 String slave = 699 "lexer grammar S;\n" + 700 "A : 'abc' {System.out.println(\"S.A\");} ;\n" + 701 "ID : 'a'..'z'+ ;\n"; 702 mkdir(tmpdir); 703 writeFile(tmpdir, "S.g", slave); 704 String master = 705 "grammar M;\n" + 706 "import S;\n" + 707 "a : A {System.out.println(\"M.a\");} ;\n" + 708 "WS : (' '|'\\n') {skip();} ;\n" ; 709 String found = execParser("M.g", master, "MParser", "MLexer", 710 "a", "abc", debug); 711 712 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 713 assertEquals("unexpected warnings: "+equeue, 0, equeue.warnings.size()); 714 715 assertEquals("S.A\nM.a\n", found); 716 } 717 testWarningForUndefinedToken()718 @Test public void testWarningForUndefinedToken() throws Exception { 719 ErrorQueue equeue = new ErrorQueue(); 720 ErrorManager.setErrorListener(equeue); 721 String slave = 722 "lexer grammar S;\n" + 723 "A : 'abc' {System.out.println(\"S.A\");} ;\n"; 724 mkdir(tmpdir); 725 writeFile(tmpdir, "S.g", slave); 726 String master = 727 "grammar M;\n" + 728 "import S;\n" + 729 "a : ABC A {System.out.println(\"M.a\");} ;\n" + 730 "WS : (' '|'\\n') {skip();} ;\n" ; 731 // A is defined in S but M should still see it and not give warning. 732 // only problem is ABC. 733 734 rawGenerateAndBuildRecognizer("M.g", master, "MParser", "MLexer", debug); 735 736 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 737 assertEquals("unexpected warnings: "+equeue, 1, equeue.warnings.size()); 738 739 String expectedError = 740 "warning(105): "+tmpdir.toString().replaceFirst("\\-[0-9]+","")+File.separator+"M.g:3:5: no lexer rule corresponding to token: ABC"; 741 assertEquals(expectedError, equeue.warnings.get(0).toString().replaceFirst("\\-[0-9]+","")); 742 } 743 744 /** Make sure that M can import S that imports T. */ test3LevelImport()745 @Test public void test3LevelImport() throws Exception { 746 ErrorQueue equeue = new ErrorQueue(); 747 ErrorManager.setErrorListener(equeue); 748 String slave = 749 "parser grammar T;\n" + 750 "a : T ;\n" ; 751 mkdir(tmpdir); 752 writeFile(tmpdir, "T.g", slave); 753 String slave2 = 754 "parser grammar S;\n" + // A, B, C token type order 755 "import T;\n" + 756 "a : S ;\n" ; 757 mkdir(tmpdir); 758 writeFile(tmpdir, "S.g", slave2); 759 760 String master = 761 "grammar M;\n" + 762 "import S;\n" + 763 "a : M ;\n" ; 764 writeFile(tmpdir, "M.g", master); 765 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 766 CompositeGrammar composite = new CompositeGrammar(); 767 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 768 composite.setDelegationRoot(g); 769 g.parseAndBuildAST(); 770 g.composite.assignTokenTypes(); 771 g.composite.defineGrammarSymbols(); 772 773 String expectedTokenIDToTypeMap = "[M=4, S=5, T=6]"; 774 String expectedStringLiteralToTypeMap = "{}"; 775 String expectedTypeToTokenList = "[M, S, T]"; 776 777 assertEquals(expectedTokenIDToTypeMap, 778 realElements(g.composite.tokenIDToTypeMap).toString()); 779 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 780 assertEquals(expectedTypeToTokenList, 781 realElements(g.composite.typeToTokenList).toString()); 782 783 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 784 785 boolean ok = 786 rawGenerateAndBuildRecognizer("M.g", master, "MParser", null, false); 787 boolean expecting = true; // should be ok 788 assertEquals(expecting, ok); 789 } 790 testBigTreeOfImports()791 @Test public void testBigTreeOfImports() throws Exception { 792 ErrorQueue equeue = new ErrorQueue(); 793 ErrorManager.setErrorListener(equeue); 794 String slave = 795 "parser grammar T;\n" + 796 "x : T ;\n" ; 797 mkdir(tmpdir); 798 writeFile(tmpdir, "T.g", slave); 799 slave = 800 "parser grammar S;\n" + 801 "import T;\n" + 802 "y : S ;\n" ; 803 mkdir(tmpdir); 804 writeFile(tmpdir, "S.g", slave); 805 806 slave = 807 "parser grammar C;\n" + 808 "i : C ;\n" ; 809 mkdir(tmpdir); 810 writeFile(tmpdir, "C.g", slave); 811 slave = 812 "parser grammar B;\n" + 813 "j : B ;\n" ; 814 mkdir(tmpdir); 815 writeFile(tmpdir, "B.g", slave); 816 slave = 817 "parser grammar A;\n" + 818 "import B,C;\n" + 819 "k : A ;\n" ; 820 mkdir(tmpdir); 821 writeFile(tmpdir, "A.g", slave); 822 823 String master = 824 "grammar M;\n" + 825 "import S,A;\n" + 826 "a : M ;\n" ; 827 writeFile(tmpdir, "M.g", master); 828 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 829 CompositeGrammar composite = new CompositeGrammar(); 830 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 831 composite.setDelegationRoot(g); 832 g.parseAndBuildAST(); 833 g.composite.assignTokenTypes(); 834 g.composite.defineGrammarSymbols(); 835 836 String expectedTokenIDToTypeMap = "[A=4, B=5, C=6, M=7, S=8, T=9]"; 837 String expectedStringLiteralToTypeMap = "{}"; 838 String expectedTypeToTokenList = "[A, B, C, M, S, T]"; 839 840 assertEquals(expectedTokenIDToTypeMap, 841 realElements(g.composite.tokenIDToTypeMap).toString()); 842 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 843 assertEquals(expectedTypeToTokenList, 844 realElements(g.composite.typeToTokenList).toString()); 845 846 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 847 848 boolean ok = 849 rawGenerateAndBuildRecognizer("M.g", master, "MParser", null, false); 850 boolean expecting = true; // should be ok 851 assertEquals(expecting, ok); 852 } 853 testRulesVisibleThroughMultilevelImport()854 @Test public void testRulesVisibleThroughMultilevelImport() throws Exception { 855 ErrorQueue equeue = new ErrorQueue(); 856 ErrorManager.setErrorListener(equeue); 857 String slave = 858 "parser grammar T;\n" + 859 "x : T ;\n" ; 860 mkdir(tmpdir); 861 writeFile(tmpdir, "T.g", slave); 862 String slave2 = 863 "parser grammar S;\n" + // A, B, C token type order 864 "import T;\n" + 865 "a : S ;\n" ; 866 mkdir(tmpdir); 867 writeFile(tmpdir, "S.g", slave2); 868 869 String master = 870 "grammar M;\n" + 871 "import S;\n" + 872 "a : M x ;\n" ; // x MUST BE VISIBLE TO M 873 writeFile(tmpdir, "M.g", master); 874 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 875 CompositeGrammar composite = new CompositeGrammar(); 876 Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); 877 composite.setDelegationRoot(g); 878 g.parseAndBuildAST(); 879 g.composite.assignTokenTypes(); 880 g.composite.defineGrammarSymbols(); 881 882 String expectedTokenIDToTypeMap = "[M=4, S=5, T=6]"; 883 String expectedStringLiteralToTypeMap = "{}"; 884 String expectedTypeToTokenList = "[M, S, T]"; 885 886 assertEquals(expectedTokenIDToTypeMap, 887 realElements(g.composite.tokenIDToTypeMap).toString()); 888 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 889 assertEquals(expectedTypeToTokenList, 890 realElements(g.composite.typeToTokenList).toString()); 891 892 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 893 } 894 testNestedComposite()895 @Test public void testNestedComposite() throws Exception { 896 // Wasn't compiling. http://www.antlr.org/jira/browse/ANTLR-438 897 ErrorQueue equeue = new ErrorQueue(); 898 ErrorManager.setErrorListener(equeue); 899 String gstr = 900 "lexer grammar L;\n" + 901 "T1: '1';\n" + 902 "T2: '2';\n" + 903 "T3: '3';\n" + 904 "T4: '4';\n" ; 905 mkdir(tmpdir); 906 writeFile(tmpdir, "L.g", gstr); 907 gstr = 908 "parser grammar G1;\n" + 909 "s: a | b;\n" + 910 "a: T1;\n" + 911 "b: T2;\n" ; 912 mkdir(tmpdir); 913 writeFile(tmpdir, "G1.g", gstr); 914 915 gstr = 916 "parser grammar G2;\n" + 917 "import G1;\n" + 918 "a: T3;\n" ; 919 mkdir(tmpdir); 920 writeFile(tmpdir, "G2.g", gstr); 921 String G3str = 922 "grammar G3;\n" + 923 "import G2;\n" + 924 "b: T4;\n" ; 925 mkdir(tmpdir); 926 writeFile(tmpdir, "G3.g", G3str); 927 928 Tool antlr = newTool(new String[] {"-lib", tmpdir}); 929 CompositeGrammar composite = new CompositeGrammar(); 930 Grammar g = new Grammar(antlr,tmpdir+"/G3.g",composite); 931 composite.setDelegationRoot(g); 932 g.parseAndBuildAST(); 933 g.composite.assignTokenTypes(); 934 g.composite.defineGrammarSymbols(); 935 936 String expectedTokenIDToTypeMap = "[T1=4, T2=5, T3=6, T4=7]"; 937 String expectedStringLiteralToTypeMap = "{}"; 938 String expectedTypeToTokenList = "[T1, T2, T3, T4]"; 939 940 assertEquals(expectedTokenIDToTypeMap, 941 realElements(g.composite.tokenIDToTypeMap).toString()); 942 assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); 943 assertEquals(expectedTypeToTokenList, 944 realElements(g.composite.typeToTokenList).toString()); 945 946 assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); 947 948 boolean ok = 949 rawGenerateAndBuildRecognizer("G3.g", G3str, "G3Parser", null, false); 950 boolean expecting = true; // should be ok 951 assertEquals(expecting, ok); 952 } 953 testHeadersPropogatedCorrectlyToImportedGrammars()954 @Test public void testHeadersPropogatedCorrectlyToImportedGrammars() throws Exception { 955 String slave = 956 "parser grammar S;\n" + 957 "a : B {System.out.print(\"S.a\");} ;\n"; 958 mkdir(tmpdir); 959 writeFile(tmpdir, "S.g", slave); 960 String master = 961 "grammar M;\n" + 962 "import S;\n" + 963 "@header{package mypackage;}\n" + 964 "@lexer::header{package mypackage;}\n" + 965 "s : a ;\n" + 966 "B : 'b' ;" + // defines B from inherited token space 967 "WS : (' '|'\\n') {skip();} ;\n" ; 968 boolean ok = antlr("M.g", "M.g", master, debug); 969 boolean expecting = true; // should be ok 970 assertEquals(expecting, ok); 971 } 972 973 }