1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 import org.antlr.Tool; 31 import org.antlr.codegen.CodeGenerator; 32 import org.antlr.tool.ErrorManager; 33 import org.antlr.tool.Grammar; 34 import org.antlr.tool.GrammarSemanticsMessage; 35 import org.junit.Ignore; 36 import org.junit.Test; 37 38 import static org.junit.Assert.*; 39 40 public class TestRewriteAST extends BaseTest { 41 protected boolean debug = false; 42 testDelete()43 @Test public void testDelete() throws Exception { 44 String grammar = 45 "grammar T;\n" + 46 "options {output=AST;}\n" + 47 "a : ID INT -> ;\n" + 48 "ID : 'a'..'z'+ ;\n" + 49 "INT : '0'..'9'+;\n" + 50 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 51 String found = execParser("T.g", grammar, "TParser", "TLexer", 52 "a", "abc 34", debug); 53 assertEquals("", found); 54 } 55 testSingleToken()56 @Test public void testSingleToken() throws Exception { 57 String grammar = 58 "grammar T;\n" + 59 "options {output=AST;}\n" + 60 "a : ID -> ID;\n" + 61 "ID : 'a'..'z'+ ;\n" + 62 "INT : '0'..'9'+;\n" + 63 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 64 String found = execParser("T.g", grammar, "TParser", "TLexer", 65 "a", "abc", debug); 66 assertEquals("abc\n", found); 67 } 68 testSingleTokenToNewNode()69 @Test public void testSingleTokenToNewNode() throws Exception { 70 String grammar = 71 "grammar T;\n" + 72 "options {output=AST;}\n" + 73 "a : ID -> ID[\"x\"];\n" + 74 "ID : 'a'..'z'+ ;\n" + 75 "INT : '0'..'9'+;\n" + 76 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 77 String found = execParser("T.g", grammar, "TParser", "TLexer", 78 "a", "abc", debug); 79 assertEquals("x\n", found); 80 } 81 testSingleTokenToNewNodeRoot()82 @Test public void testSingleTokenToNewNodeRoot() throws Exception { 83 String grammar = 84 "grammar T;\n" + 85 "options {output=AST;}\n" + 86 "a : ID -> ^(ID[\"x\"] INT);\n" + 87 "ID : 'a'..'z'+ ;\n" + 88 "INT : '0'..'9'+;\n" + 89 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 90 String found = execParser("T.g", grammar, "TParser", "TLexer", 91 "a", "abc", debug); 92 assertEquals("(x INT)\n", found); 93 } 94 testSingleTokenToNewNode2()95 @Test public void testSingleTokenToNewNode2() throws Exception { 96 // Allow creation of new nodes w/o args. 97 String grammar = 98 "grammar TT;\n" + 99 "options {output=AST;}\n" + 100 "a : ID -> ID[ ];\n" + 101 "ID : 'a'..'z'+ ;\n" + 102 "INT : '0'..'9'+;\n" + 103 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 104 String found = execParser("TT.g", grammar, "TTParser", "TTLexer", 105 "a", "abc", debug); 106 assertEquals("ID\n", found); 107 } 108 testSingleCharLiteral()109 @Test public void testSingleCharLiteral() throws Exception { 110 String grammar = 111 "grammar T;\n" + 112 "options {output=AST;}\n" + 113 "a : 'c' -> 'c';\n" + 114 "ID : 'a'..'z'+ ;\n" + 115 "INT : '0'..'9'+;\n" + 116 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 117 String found = execParser("T.g", grammar, "TParser", "TLexer", 118 "a", "c", debug); 119 assertEquals("c\n", found); 120 } 121 testSingleStringLiteral()122 @Test public void testSingleStringLiteral() throws Exception { 123 String grammar = 124 "grammar T;\n" + 125 "options {output=AST;}\n" + 126 "a : 'ick' -> 'ick';\n" + 127 "ID : 'a'..'z'+ ;\n" + 128 "INT : '0'..'9'+;\n" + 129 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 130 String found = execParser("T.g", grammar, "TParser", "TLexer", 131 "a", "ick", debug); 132 assertEquals("ick\n", found); 133 } 134 testSingleRule()135 @Test public void testSingleRule() throws Exception { 136 String grammar = 137 "grammar T;\n" + 138 "options {output=AST;}\n" + 139 "a : b -> b;\n" + 140 "b : ID ;\n" + 141 "ID : 'a'..'z'+ ;\n" + 142 "INT : '0'..'9'+;\n" + 143 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 144 String found = execParser("T.g", grammar, "TParser", "TLexer", 145 "a", "abc", debug); 146 assertEquals("abc\n", found); 147 } 148 testReorderTokens()149 @Test public void testReorderTokens() throws Exception { 150 String grammar = 151 "grammar T;\n" + 152 "options {output=AST;}\n" + 153 "a : ID INT -> INT ID;\n" + 154 "ID : 'a'..'z'+ ;\n" + 155 "INT : '0'..'9'+;\n" + 156 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 157 String found = execParser("T.g", grammar, "TParser", "TLexer", 158 "a", "abc 34", debug); 159 assertEquals("34 abc\n", found); 160 } 161 testReorderTokenAndRule()162 @Test public void testReorderTokenAndRule() throws Exception { 163 String grammar = 164 "grammar T;\n" + 165 "options {output=AST;}\n" + 166 "a : b INT -> INT b;\n" + 167 "b : ID ;\n" + 168 "ID : 'a'..'z'+ ;\n" + 169 "INT : '0'..'9'+;\n" + 170 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 171 String found = execParser("T.g", grammar, "TParser", "TLexer", 172 "a", "abc 34", debug); 173 assertEquals("34 abc\n", found); 174 } 175 testTokenTree()176 @Test public void testTokenTree() throws Exception { 177 String grammar = 178 "grammar T;\n" + 179 "options {output=AST;}\n" + 180 "a : ID INT -> ^(INT ID);\n" + 181 "ID : 'a'..'z'+ ;\n" + 182 "INT : '0'..'9'+;\n" + 183 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 184 String found = execParser("T.g", grammar, "TParser", "TLexer", 185 "a", "abc 34", debug); 186 assertEquals("(34 abc)\n", found); 187 } 188 testTokenTreeAfterOtherStuff()189 @Test public void testTokenTreeAfterOtherStuff() throws Exception { 190 String grammar = 191 "grammar T;\n" + 192 "options {output=AST;}\n" + 193 "a : 'void' ID INT -> 'void' ^(INT ID);\n" + 194 "ID : 'a'..'z'+ ;\n" + 195 "INT : '0'..'9'+;\n" + 196 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 197 String found = execParser("T.g", grammar, "TParser", "TLexer", 198 "a", "void abc 34", debug); 199 assertEquals("void (34 abc)\n", found); 200 } 201 testNestedTokenTreeWithOuterLoop()202 @Test public void testNestedTokenTreeWithOuterLoop() throws Exception { 203 // verify that ID and INT both iterate over outer index variable 204 String grammar = 205 "grammar T;\n" + 206 "options {output=AST;}\n" + 207 "tokens {DUH;}\n" + 208 "a : ID INT ID INT -> ^( DUH ID ^( DUH INT) )+ ;\n" + 209 "ID : 'a'..'z'+ ;\n" + 210 "INT : '0'..'9'+;\n" + 211 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 212 String found = execParser("T.g", grammar, "TParser", "TLexer", 213 "a", "a 1 b 2", debug); 214 assertEquals("(DUH a (DUH 1)) (DUH b (DUH 2))\n", found); 215 } 216 testOptionalSingleToken()217 @Test public void testOptionalSingleToken() throws Exception { 218 String grammar = 219 "grammar T;\n" + 220 "options {output=AST;}\n" + 221 "a : ID -> ID? ;\n" + 222 "ID : 'a'..'z'+ ;\n" + 223 "INT : '0'..'9'+;\n" + 224 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 225 String found = execParser("T.g", grammar, "TParser", "TLexer", 226 "a", "abc", debug); 227 assertEquals("abc\n", found); 228 } 229 testClosureSingleToken()230 @Test public void testClosureSingleToken() throws Exception { 231 String grammar = 232 "grammar T;\n" + 233 "options {output=AST;}\n" + 234 "a : ID ID -> ID* ;\n" + 235 "ID : 'a'..'z'+ ;\n" + 236 "INT : '0'..'9'+;\n" + 237 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 238 String found = execParser("T.g", grammar, "TParser", "TLexer", 239 "a", "a b", debug); 240 assertEquals("a b\n", found); 241 } 242 testPositiveClosureSingleToken()243 @Test public void testPositiveClosureSingleToken() throws Exception { 244 String grammar = 245 "grammar T;\n" + 246 "options {output=AST;}\n" + 247 "a : ID ID -> ID+ ;\n" + 248 "ID : 'a'..'z'+ ;\n" + 249 "INT : '0'..'9'+;\n" + 250 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 251 String found = execParser("T.g", grammar, "TParser", "TLexer", 252 "a", "a b", debug); 253 assertEquals("a b\n", found); 254 } 255 testOptionalSingleRule()256 @Test public void testOptionalSingleRule() throws Exception { 257 String grammar = 258 "grammar T;\n" + 259 "options {output=AST;}\n" + 260 "a : b -> b?;\n" + 261 "b : ID ;\n" + 262 "ID : 'a'..'z'+ ;\n" + 263 "INT : '0'..'9'+;\n" + 264 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 265 String found = execParser("T.g", grammar, "TParser", "TLexer", 266 "a", "abc", debug); 267 assertEquals("abc\n", found); 268 } 269 testClosureSingleRule()270 @Test public void testClosureSingleRule() throws Exception { 271 String grammar = 272 "grammar T;\n" + 273 "options {output=AST;}\n" + 274 "a : b b -> b*;\n" + 275 "b : ID ;\n" + 276 "ID : 'a'..'z'+ ;\n" + 277 "INT : '0'..'9'+;\n" + 278 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 279 String found = execParser("T.g", grammar, "TParser", "TLexer", 280 "a", "a b", debug); 281 assertEquals("a b\n", found); 282 } 283 testClosureOfLabel()284 @Test public void testClosureOfLabel() throws Exception { 285 String grammar = 286 "grammar T;\n" + 287 "options {output=AST;}\n" + 288 "a : x+=b x+=b -> $x*;\n" + 289 "b : ID ;\n" + 290 "ID : 'a'..'z'+ ;\n" + 291 "INT : '0'..'9'+;\n" + 292 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 293 String found = execParser("T.g", grammar, "TParser", "TLexer", 294 "a", "a b", debug); 295 assertEquals("a b\n", found); 296 } 297 testOptionalLabelNoListLabel()298 @Test public void testOptionalLabelNoListLabel() throws Exception { 299 String grammar = 300 "grammar T;\n" + 301 "options {output=AST;}\n" + 302 "a : (x=ID)? -> $x?;\n" + 303 "ID : 'a'..'z'+ ;\n" + 304 "INT : '0'..'9'+;\n" + 305 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 306 String found = execParser("T.g", grammar, "TParser", "TLexer", 307 "a", "a", debug); 308 assertEquals("a\n", found); 309 } 310 testPositiveClosureSingleRule()311 @Test public void testPositiveClosureSingleRule() throws Exception { 312 String grammar = 313 "grammar T;\n" + 314 "options {output=AST;}\n" + 315 "a : b b -> b+;\n" + 316 "b : ID ;\n" + 317 "ID : 'a'..'z'+ ;\n" + 318 "INT : '0'..'9'+;\n" + 319 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 320 String found = execParser("T.g", grammar, "TParser", "TLexer", 321 "a", "a b", debug); 322 assertEquals("a b\n", found); 323 } 324 testSinglePredicateT()325 @Test public void testSinglePredicateT() throws Exception { 326 String grammar = 327 "grammar T;\n" + 328 "options {output=AST;}\n" + 329 "a : ID -> {true}? ID -> ;\n" + 330 "ID : 'a'..'z'+ ;\n" + 331 "INT : '0'..'9'+;\n" + 332 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 333 String found = execParser("T.g", grammar, "TParser", "TLexer", 334 "a", "abc", debug); 335 assertEquals("abc\n", found); 336 } 337 testSinglePredicateF()338 @Test public void testSinglePredicateF() throws Exception { 339 String grammar = 340 "grammar T;\n" + 341 "options {output=AST;}\n" + 342 "a : ID -> {false}? ID -> ;\n" + 343 "ID : 'a'..'z'+ ;\n" + 344 "INT : '0'..'9'+;\n" + 345 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 346 String found = execParser("T.g", grammar, "TParser", "TLexer", 347 "a", "abc", debug); 348 assertEquals("", found); 349 } 350 testMultiplePredicate()351 @Test public void testMultiplePredicate() throws Exception { 352 String grammar = 353 "grammar T;\n" + 354 "options {output=AST;}\n" + 355 "a : ID INT -> {false}? ID\n" + 356 " -> {true}? INT\n" + 357 " -> \n" + 358 " ;\n" + 359 "ID : 'a'..'z'+ ;\n" + 360 "INT : '0'..'9'+;\n" + 361 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 362 String found = execParser("T.g", grammar, "TParser", "TLexer", 363 "a", "a 2", debug); 364 assertEquals("2\n", found); 365 } 366 testMultiplePredicateTrees()367 @Test public void testMultiplePredicateTrees() throws Exception { 368 String grammar = 369 "grammar T;\n" + 370 "options {output=AST;}\n" + 371 "a : ID INT -> {false}? ^(ID INT)\n" + 372 " -> {true}? ^(INT ID)\n" + 373 " -> ID\n" + 374 " ;\n" + 375 "ID : 'a'..'z'+ ;\n" + 376 "INT : '0'..'9'+;\n" + 377 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 378 String found = execParser("T.g", grammar, "TParser", "TLexer", 379 "a", "a 2", debug); 380 assertEquals("(2 a)\n", found); 381 } 382 testSimpleTree()383 @Test public void testSimpleTree() throws Exception { 384 String grammar = 385 "grammar T;\n" + 386 "options {output=AST;}\n" + 387 "a : op INT -> ^(op INT);\n" + 388 "op : '+'|'-' ;\n" + 389 "ID : 'a'..'z'+ ;\n" + 390 "INT : '0'..'9'+;\n" + 391 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 392 String found = execParser("T.g", grammar, "TParser", "TLexer", 393 "a", "-34", debug); 394 assertEquals("(- 34)\n", found); 395 } 396 testSimpleTree2()397 @Test public void testSimpleTree2() throws Exception { 398 String grammar = 399 "grammar T;\n" + 400 "options {output=AST;}\n" + 401 "a : op INT -> ^(INT op);\n" + 402 "op : '+'|'-' ;\n" + 403 "ID : 'a'..'z'+ ;\n" + 404 "INT : '0'..'9'+;\n" + 405 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 406 String found = execParser("T.g", grammar, "TParser", "TLexer", 407 "a", "+ 34", debug); 408 assertEquals("(34 +)\n", found); 409 } 410 411 testNestedTrees()412 @Test public void testNestedTrees() throws Exception { 413 String grammar = 414 "grammar T;\n" + 415 "options {output=AST;}\n" + 416 "a : 'var' (ID ':' type ';')+ -> ^('var' ^(':' ID type)+) ;\n" + 417 "type : 'int' | 'float' ;\n" + 418 "ID : 'a'..'z'+ ;\n" + 419 "INT : '0'..'9'+;\n" + 420 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 421 String found = execParser("T.g", grammar, "TParser", "TLexer", 422 "a", "var a:int; b:float;", debug); 423 assertEquals("(var (: a int) (: b float))\n", found); 424 } 425 testImaginaryTokenCopy()426 @Test public void testImaginaryTokenCopy() throws Exception { 427 String grammar = 428 "grammar T;\n" + 429 "options {output=AST;}\n" + 430 "tokens {VAR;}\n" + 431 "a : ID (',' ID)*-> ^(VAR ID)+ ;\n" + 432 "type : 'int' | 'float' ;\n" + 433 "ID : 'a'..'z'+ ;\n" + 434 "INT : '0'..'9'+;\n" + 435 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 436 String found = execParser("T.g", grammar, "TParser", "TLexer", 437 "a", "a,b,c", debug); 438 assertEquals("(VAR a) (VAR b) (VAR c)\n", found); 439 } 440 testTokenUnreferencedOnLeftButDefined()441 @Test public void testTokenUnreferencedOnLeftButDefined() throws Exception { 442 String grammar = 443 "grammar T;\n" + 444 "options {output=AST;}\n" + 445 "tokens {VAR;}\n" + 446 "a : b -> ID ;\n" + 447 "b : ID ;\n"+ 448 "ID : 'a'..'z'+ ;\n" + 449 "INT : '0'..'9'+;\n" + 450 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 451 String found = execParser("T.g", grammar, "TParser", "TLexer", 452 "a", "a", debug); 453 assertEquals("ID\n", found); 454 } 455 testImaginaryTokenCopySetText()456 @Test public void testImaginaryTokenCopySetText() throws Exception { 457 String grammar = 458 "grammar T;\n" + 459 "options {output=AST;}\n" + 460 "tokens {VAR;}\n" + 461 "a : ID (',' ID)*-> ^(VAR[\"var\"] ID)+ ;\n" + 462 "type : 'int' | 'float' ;\n" + 463 "ID : 'a'..'z'+ ;\n" + 464 "INT : '0'..'9'+;\n" + 465 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 466 String found = execParser("T.g", grammar, "TParser", "TLexer", 467 "a", "a,b,c", debug); 468 assertEquals("(var a) (var b) (var c)\n", found); 469 } 470 testImaginaryTokenNoCopyFromToken()471 @Test public void testImaginaryTokenNoCopyFromToken() throws Exception { 472 String grammar = 473 "grammar T;\n" + 474 "options {output=AST;}\n" + 475 "tokens {BLOCK;}\n" + 476 "a : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;\n" + 477 "type : 'int' | 'float' ;\n" + 478 "ID : 'a'..'z'+ ;\n" + 479 "INT : '0'..'9'+;\n" + 480 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 481 String found = execParser("T.g", grammar, "TParser", "TLexer", 482 "a", "{a b c}", debug); 483 assertEquals("({ a b c)\n", found); 484 } 485 testImaginaryTokenNoCopyFromTokenSetText()486 @Test public void testImaginaryTokenNoCopyFromTokenSetText() throws Exception { 487 String grammar = 488 "grammar T;\n" + 489 "options {output=AST;}\n" + 490 "tokens {BLOCK;}\n" + 491 "a : lc='{' ID+ '}' -> ^(BLOCK[$lc,\"block\"] ID+) ;\n" + 492 "type : 'int' | 'float' ;\n" + 493 "ID : 'a'..'z'+ ;\n" + 494 "INT : '0'..'9'+;\n" + 495 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 496 String found = execParser("T.g", grammar, "TParser", "TLexer", 497 "a", "{a b c}", debug); 498 assertEquals("(block a b c)\n", found); 499 } 500 testMixedRewriteAndAutoAST()501 @Test public void testMixedRewriteAndAutoAST() throws Exception { 502 String grammar = 503 "grammar T;\n" + 504 "options {output=AST;}\n" + 505 "tokens {BLOCK;}\n" + 506 "a : b b^ ;\n" + // 2nd b matches only an INT; can make it root 507 "b : ID INT -> INT ID\n" + 508 " | INT\n" + 509 " ;\n" + 510 "ID : 'a'..'z'+ ;\n" + 511 "INT : '0'..'9'+;\n" + 512 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 513 String found = execParser("T.g", grammar, "TParser", "TLexer", 514 "a", "a 1 2", debug); 515 assertEquals("(2 1 a)\n", found); 516 } 517 testSubruleWithRewrite()518 @Test public void testSubruleWithRewrite() throws Exception { 519 String grammar = 520 "grammar T;\n" + 521 "options {output=AST;}\n" + 522 "tokens {BLOCK;}\n" + 523 "a : b b ;\n" + 524 "b : (ID INT -> INT ID | INT INT -> INT+ )\n" + 525 " ;\n" + 526 "ID : 'a'..'z'+ ;\n" + 527 "INT : '0'..'9'+;\n" + 528 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 529 String found = execParser("T.g", grammar, "TParser", "TLexer", 530 "a", "a 1 2 3", debug); 531 assertEquals("1 a 2 3\n", found); 532 } 533 testSubruleWithRewrite2()534 @Test public void testSubruleWithRewrite2() throws Exception { 535 String grammar = 536 "grammar T;\n" + 537 "options {output=AST;}\n" + 538 "tokens {TYPE;}\n" + 539 "a : b b ;\n" + 540 "b : 'int'\n" + 541 " ( ID -> ^(TYPE 'int' ID)\n" + 542 " | ID '=' INT -> ^(TYPE 'int' ID INT)\n" + 543 " )\n" + 544 " ';'\n" + 545 " ;\n" + 546 "ID : 'a'..'z'+ ;\n" + 547 "INT : '0'..'9'+;\n" + 548 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 549 String found = execParser("T.g", grammar, "TParser", "TLexer", 550 "a", "int a; int b=3;", debug); 551 assertEquals("(TYPE int a) (TYPE int b 3)\n", found); 552 } 553 testNestedRewriteShutsOffAutoAST()554 @Test public void testNestedRewriteShutsOffAutoAST() throws Exception { 555 String grammar = 556 "grammar T;\n" + 557 "options {output=AST;}\n" + 558 "tokens {BLOCK;}\n" + 559 "a : b b ;\n" + 560 "b : ID ( ID (last=ID -> $last)+ ) ';'\n" + // get last ID 561 " | INT\n" + // should still get auto AST construction 562 " ;\n" + 563 "ID : 'a'..'z'+ ;\n" + 564 "INT : '0'..'9'+;\n" + 565 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 566 String found = execParser("T.g", grammar, "TParser", "TLexer", 567 "a", "a b c d; 42", debug); 568 assertEquals("d 42\n", found); 569 } 570 testRewriteActions()571 @Test public void testRewriteActions() throws Exception { 572 String grammar = 573 "grammar T;\n" + 574 "options {output=AST;}\n" + 575 "a : atom -> ^({adaptor.create(INT,\"9\")} atom) ;\n" + 576 "atom : INT ;\n" + 577 "ID : 'a'..'z'+ ;\n" + 578 "INT : '0'..'9'+;\n" + 579 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 580 String found = execParser("T.g", grammar, "TParser", "TLexer", 581 "a", "3", debug); 582 assertEquals("(9 3)\n", found); 583 } 584 testRewriteActions2()585 @Test public void testRewriteActions2() throws Exception { 586 String grammar = 587 "grammar T;\n" + 588 "options {output=AST;}\n" + 589 "a : atom -> {adaptor.create(INT,\"9\")} atom ;\n" + 590 "atom : INT ;\n" + 591 "ID : 'a'..'z'+ ;\n" + 592 "INT : '0'..'9'+;\n" + 593 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 594 String found = execParser("T.g", grammar, "TParser", "TLexer", 595 "a", "3", debug); 596 assertEquals("9 3\n", found); 597 } 598 testRefToOldValue()599 @Test public void testRefToOldValue() throws Exception { 600 String grammar = 601 "grammar T;\n" + 602 "options {output=AST;}\n" + 603 "tokens {BLOCK;}\n" + 604 "a : (atom -> atom) (op='+' r=atom -> ^($op $a $r) )* ;\n" + 605 "atom : INT ;\n" + 606 "ID : 'a'..'z'+ ;\n" + 607 "INT : '0'..'9'+;\n" + 608 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 609 String found = execParser("T.g", grammar, "TParser", "TLexer", 610 "a", "3+4+5", debug); 611 assertEquals("(+ (+ 3 4) 5)\n", found); 612 } 613 testCopySemanticsForRules()614 @Test public void testCopySemanticsForRules() throws Exception { 615 String grammar = 616 "grammar T;\n" + 617 "options {output=AST;}\n" + 618 "tokens {BLOCK;}\n" + 619 "a : atom -> ^(atom atom) ;\n" + // NOT CYCLE! (dup atom) 620 "atom : INT ;\n" + 621 "ID : 'a'..'z'+ ;\n" + 622 "INT : '0'..'9'+;\n" + 623 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 624 String found = execParser("T.g", grammar, "TParser", "TLexer", 625 "a", "3", debug); 626 assertEquals("(3 3)\n", found); 627 } 628 testCopySemanticsForRules2()629 @Test public void testCopySemanticsForRules2() throws Exception { 630 // copy type as a root for each invocation of (...)+ in rewrite 631 String grammar = 632 "grammar T;\n" + 633 "options {output=AST;}\n" + 634 "a : type ID (',' ID)* ';' -> ^(type ID)+ ;\n" + 635 "type : 'int' ;\n" + 636 "ID : 'a'..'z'+ ;\n" + 637 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 638 String found = execParser("T.g", grammar, "TParser", "TLexer", 639 "a", "int a,b,c;", debug); 640 assertEquals("(int a) (int b) (int c)\n", found); 641 } 642 testCopySemanticsForRules3()643 @Test public void testCopySemanticsForRules3() throws Exception { 644 // copy type *and* modifier even though it's optional 645 // for each invocation of (...)+ in rewrite 646 String grammar = 647 "grammar T;\n" + 648 "options {output=AST;}\n" + 649 "a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)+ ;\n" + 650 "type : 'int' ;\n" + 651 "modifier : 'public' ;\n" + 652 "ID : 'a'..'z'+ ;\n" + 653 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 654 String found = execParser("T.g", grammar, "TParser", "TLexer", 655 "a", "public int a,b,c;", debug); 656 assertEquals("(int public a) (int public b) (int public c)\n", found); 657 } 658 testCopySemanticsForRules3Double()659 @Test public void testCopySemanticsForRules3Double() throws Exception { 660 // copy type *and* modifier even though it's optional 661 // for each invocation of (...)+ in rewrite 662 String grammar = 663 "grammar T;\n" + 664 "options {output=AST;}\n" + 665 "a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)+ ^(type modifier? ID)+ ;\n" + 666 "type : 'int' ;\n" + 667 "modifier : 'public' ;\n" + 668 "ID : 'a'..'z'+ ;\n" + 669 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 670 String found = execParser("T.g", grammar, "TParser", "TLexer", 671 "a", "public int a,b,c;", debug); 672 assertEquals("(int public a) (int public b) (int public c) (int public a) (int public b) (int public c)\n", found); 673 } 674 testCopySemanticsForRules4()675 @Test public void testCopySemanticsForRules4() throws Exception { 676 // copy type *and* modifier even though it's optional 677 // for each invocation of (...)+ in rewrite 678 String grammar = 679 "grammar T;\n" + 680 "options {output=AST;}\n" + 681 "tokens {MOD;}\n" + 682 "a : modifier? type ID (',' ID)* ';' -> ^(type ^(MOD modifier)? ID)+ ;\n" + 683 "type : 'int' ;\n" + 684 "modifier : 'public' ;\n" + 685 "ID : 'a'..'z'+ ;\n" + 686 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 687 String found = execParser("T.g", grammar, "TParser", "TLexer", 688 "a", "public int a,b,c;", debug); 689 assertEquals("(int (MOD public) a) (int (MOD public) b) (int (MOD public) c)\n", found); 690 } 691 testCopySemanticsLists()692 @Test public void testCopySemanticsLists() throws Exception { 693 String grammar = 694 "grammar T;\n" + 695 "options {output=AST;}\n" + 696 "tokens {MOD;}\n" + 697 "a : ID (',' ID)* ';' -> ID+ ID+ ;\n"+ 698 "ID : 'a'..'z'+ ;\n" + 699 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 700 String found = execParser("T.g", grammar, "TParser", "TLexer", 701 "a", "a,b,c;", debug); 702 assertEquals("a b c a b c\n", found); 703 } 704 testCopyRuleLabel()705 @Test public void testCopyRuleLabel() throws Exception { 706 String grammar = 707 "grammar T;\n" + 708 "options {output=AST;}\n" + 709 "tokens {BLOCK;}\n" + 710 "a : x=b -> $x $x;\n"+ 711 "b : ID ;\n"+ 712 "ID : 'a'..'z'+ ;\n" + 713 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 714 String found = execParser("T.g", grammar, "TParser", "TLexer", 715 "a", "a", debug); 716 assertEquals("a a\n", found); 717 } 718 testCopyRuleLabel2()719 @Test public void testCopyRuleLabel2() throws Exception { 720 String grammar = 721 "grammar T;\n" + 722 "options {output=AST;}\n" + 723 "tokens {BLOCK;}\n" + 724 "a : x=b -> ^($x $x);\n"+ 725 "b : ID ;\n"+ 726 "ID : 'a'..'z'+ ;\n" + 727 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 728 String found = execParser("T.g", grammar, "TParser", "TLexer", 729 "a", "a", debug); 730 assertEquals("(a a)\n", found); 731 } 732 testQueueingOfTokens()733 @Test public void testQueueingOfTokens() throws Exception { 734 String grammar = 735 "grammar T;\n" + 736 "options {output=AST;}\n" + 737 "a : 'int' ID (',' ID)* ';' -> ^('int' ID+) ;\n" + 738 "op : '+'|'-' ;\n" + 739 "ID : 'a'..'z'+ ;\n" + 740 "INT : '0'..'9'+;\n" + 741 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 742 String found = execParser("T.g", grammar, "TParser", "TLexer", 743 "a", "int a,b,c;", debug); 744 assertEquals("(int a b c)\n", found); 745 } 746 testCopyOfTokens()747 @Test public void testCopyOfTokens() throws Exception { 748 String grammar = 749 "grammar T;\n" + 750 "options {output=AST;}\n" + 751 "a : 'int' ID ';' -> 'int' ID 'int' ID ;\n" + 752 "op : '+'|'-' ;\n" + 753 "ID : 'a'..'z'+ ;\n" + 754 "INT : '0'..'9'+;\n" + 755 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 756 String found = execParser("T.g", grammar, "TParser", "TLexer", 757 "a", "int a;", debug); 758 assertEquals("int a int a\n", found); 759 } 760 testTokenCopyInLoop()761 @Test public void testTokenCopyInLoop() throws Exception { 762 String grammar = 763 "grammar T;\n" + 764 "options {output=AST;}\n" + 765 "a : 'int' ID (',' ID)* ';' -> ^('int' ID)+ ;\n" + 766 "op : '+'|'-' ;\n" + 767 "ID : 'a'..'z'+ ;\n" + 768 "INT : '0'..'9'+;\n" + 769 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 770 String found = execParser("T.g", grammar, "TParser", "TLexer", 771 "a", "int a,b,c;", debug); 772 assertEquals("(int a) (int b) (int c)\n", found); 773 } 774 testTokenCopyInLoopAgainstTwoOthers()775 @Test public void testTokenCopyInLoopAgainstTwoOthers() throws Exception { 776 // must smear 'int' copies across as root of multiple trees 777 String grammar = 778 "grammar T;\n" + 779 "options {output=AST;}\n" + 780 "a : 'int' ID ':' INT (',' ID ':' INT)* ';' -> ^('int' ID INT)+ ;\n" + 781 "op : '+'|'-' ;\n" + 782 "ID : 'a'..'z'+ ;\n" + 783 "INT : '0'..'9'+;\n" + 784 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 785 String found = execParser("T.g", grammar, "TParser", "TLexer", 786 "a", "int a:1,b:2,c:3;", debug); 787 assertEquals("(int a 1) (int b 2) (int c 3)\n", found); 788 } 789 testListRefdOneAtATime()790 @Test public void testListRefdOneAtATime() throws Exception { 791 String grammar = 792 "grammar T;\n" + 793 "options {output=AST;}\n" + 794 "a : ID+ -> ID ID ID ;\n" + // works if 3 input IDs 795 "op : '+'|'-' ;\n" + 796 "ID : 'a'..'z'+ ;\n" + 797 "INT : '0'..'9'+;\n" + 798 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 799 String found = execParser("T.g", grammar, "TParser", "TLexer", 800 "a", "a b c", debug); 801 assertEquals("a b c\n", found); 802 } 803 testSplitListWithLabels()804 @Test public void testSplitListWithLabels() throws Exception { 805 String grammar = 806 "grammar T;\n" + 807 "options {output=AST;}\n" + 808 "tokens {VAR;}\n"+ 809 "a : first=ID others+=ID* -> $first VAR $others+ ;\n" + 810 "op : '+'|'-' ;\n" + 811 "ID : 'a'..'z'+ ;\n" + 812 "INT : '0'..'9'+;\n" + 813 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 814 String found = execParser("T.g", grammar, "TParser", "TLexer", 815 "a", "a b c", debug); 816 assertEquals("a VAR b c\n", found); 817 } 818 testComplicatedMelange()819 @Test public void testComplicatedMelange() throws Exception { 820 String grammar = 821 "grammar T;\n" + 822 "options {output=AST;}\n" + 823 "tokens {BLOCK;}\n" + 824 "a : A A b=B B b=B c+=C C c+=C D {String s=$D.text;} -> A+ B+ C+ D ;\n" + 825 "type : 'int' | 'float' ;\n" + 826 "A : 'a' ;\n" + 827 "B : 'b' ;\n" + 828 "C : 'c' ;\n" + 829 "D : 'd' ;\n" + 830 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 831 String found = execParser("T.g", grammar, "TParser", "TLexer", 832 "a", "a a b b b c c c d", debug); 833 assertEquals("a a b b b c c c d\n", found); 834 } 835 testRuleLabel()836 @Test public void testRuleLabel() throws Exception { 837 String grammar = 838 "grammar T;\n" + 839 "options {output=AST;}\n" + 840 "tokens {BLOCK;}\n" + 841 "a : x=b -> $x;\n"+ 842 "b : ID ;\n"+ 843 "ID : 'a'..'z'+ ;\n" + 844 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 845 String found = execParser("T.g", grammar, "TParser", "TLexer", 846 "a", "a", debug); 847 assertEquals("a\n", found); 848 } 849 testAmbiguousRule()850 @Test public void testAmbiguousRule() throws Exception { 851 String grammar = 852 "grammar T;\n" + 853 "options {output=AST;}\n" + 854 "a : ID a -> a | INT ;\n"+ 855 "ID : 'a'..'z'+ ;\n" + 856 "INT: '0'..'9'+ ;\n" + 857 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 858 String found = execParser("T.g", grammar, "TParser", "TLexer", 859 "a", "abc 34", debug); 860 assertEquals("34\n", found); 861 } 862 testWeirdRuleRef()863 @Test public void testWeirdRuleRef() throws Exception { 864 ErrorQueue equeue = new ErrorQueue(); 865 ErrorManager.setErrorListener(equeue); 866 String grammar = 867 "grammar T;\n" + 868 "options {output=AST;}\n" + 869 "a : ID a -> $a | INT ;\n"+ 870 "ID : 'a'..'z'+ ;\n" + 871 "INT: '0'..'9'+ ;\n" + 872 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 873 874 Grammar g = new Grammar(grammar); 875 Tool antlr = newTool(); 876 antlr.setOutputDirectory(null); // write to /dev/null 877 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 878 g.setCodeGenerator(generator); 879 generator.genRecognizer(); 880 881 // $a is ambig; is it previous root or ref to a ref in alt? 882 assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); 883 } 884 testRuleListLabel()885 @Test public void testRuleListLabel() throws Exception { 886 String grammar = 887 "grammar T;\n" + 888 "options {output=AST;}\n" + 889 "tokens {BLOCK;}\n" + 890 "a : x+=b x+=b -> $x+;\n"+ 891 "b : ID ;\n"+ 892 "ID : 'a'..'z'+ ;\n" + 893 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 894 String found = execParser("T.g", grammar, "TParser", "TLexer", 895 "a", "a b", debug); 896 assertEquals("a b\n", found); 897 } 898 testRuleListLabel2()899 @Test public void testRuleListLabel2() throws Exception { 900 String grammar = 901 "grammar T;\n" + 902 "options {output=AST;}\n" + 903 "tokens {BLOCK;}\n" + 904 "a : x+=b x+=b -> $x $x*;\n"+ 905 "b : ID ;\n"+ 906 "ID : 'a'..'z'+ ;\n" + 907 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 908 String found = execParser("T.g", grammar, "TParser", "TLexer", 909 "a", "a b", debug); 910 assertEquals("a b\n", found); 911 } 912 testOptional()913 @Test public void testOptional() throws Exception { 914 String grammar = 915 "grammar T;\n" + 916 "options {output=AST;}\n" + 917 "tokens {BLOCK;}\n" + 918 "a : x=b (y=b)? -> $x $y?;\n"+ 919 "b : ID ;\n"+ 920 "ID : 'a'..'z'+ ;\n" + 921 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 922 String found = execParser("T.g", grammar, "TParser", "TLexer", 923 "a", "a", debug); 924 assertEquals("a\n", found); 925 } 926 testOptional2()927 @Test public void testOptional2() throws Exception { 928 String grammar = 929 "grammar T;\n" + 930 "options {output=AST;}\n" + 931 "tokens {BLOCK;}\n" + 932 "a : x=ID (y=b)? -> $x $y?;\n"+ 933 "b : ID ;\n"+ 934 "ID : 'a'..'z'+ ;\n" + 935 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 936 String found = execParser("T.g", grammar, "TParser", "TLexer", 937 "a", "a b", debug); 938 assertEquals("a b\n", found); 939 } 940 testOptional3()941 @Test public void testOptional3() throws Exception { 942 String grammar = 943 "grammar T;\n" + 944 "options {output=AST;}\n" + 945 "tokens {BLOCK;}\n" + 946 "a : x=ID (y=b)? -> ($x $y)?;\n"+ 947 "b : ID ;\n"+ 948 "ID : 'a'..'z'+ ;\n" + 949 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 950 String found = execParser("T.g", grammar, "TParser", "TLexer", 951 "a", "a b", debug); 952 assertEquals("a b\n", found); 953 } 954 testOptional4()955 @Test public void testOptional4() throws Exception { 956 String grammar = 957 "grammar T;\n" + 958 "options {output=AST;}\n" + 959 "tokens {BLOCK;}\n" + 960 "a : x+=ID (y=b)? -> ($x $y)?;\n"+ 961 "b : ID ;\n"+ 962 "ID : 'a'..'z'+ ;\n" + 963 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 964 String found = execParser("T.g", grammar, "TParser", "TLexer", 965 "a", "a b", debug); 966 assertEquals("a b\n", found); 967 } 968 testOptional5()969 @Test public void testOptional5() throws Exception { 970 String grammar = 971 "grammar T;\n" + 972 "options {output=AST;}\n" + 973 "tokens {BLOCK;}\n" + 974 "a : ID -> ID? ;\n"+ // match an ID to optional ID 975 "b : ID ;\n"+ 976 "ID : 'a'..'z'+ ;\n" + 977 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 978 String found = execParser("T.g", grammar, "TParser", "TLexer", 979 "a", "a", debug); 980 assertEquals("a\n", found); 981 } 982 testArbitraryExprType()983 @Test public void testArbitraryExprType() throws Exception { 984 String grammar = 985 "grammar T;\n" + 986 "options {output=AST;}\n" + 987 "tokens {BLOCK;}\n" + 988 "a : x+=b x+=b -> {new CommonTree()};\n"+ 989 "b : ID ;\n"+ 990 "ID : 'a'..'z'+ ;\n" + 991 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 992 String found = execParser("T.g", grammar, "TParser", "TLexer", 993 "a", "a b", debug); 994 assertEquals("", found); 995 } 996 testSet()997 @Test public void testSet() throws Exception { 998 String grammar = 999 "grammar T;\n" + 1000 "options { output = AST; } \n" + 1001 "a: (INT|ID)+ -> INT+ ID+ ;\n" + 1002 "INT: '0'..'9'+;\n" + 1003 "ID : 'a'..'z'+;\n" + 1004 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1005 String found = execParser("T.g", grammar, "TParser", "TLexer", 1006 "a", "2 a 34 de", debug); 1007 assertEquals("2 34 a de\n", found); 1008 } 1009 testSet2()1010 @Test public void testSet2() throws Exception { 1011 String grammar = 1012 "grammar T;\n" + 1013 "options { output = AST; } \n" + 1014 "a: (INT|ID) -> INT? ID? ;\n" + 1015 "INT: '0'..'9'+;\n" + 1016 "ID : 'a'..'z'+;\n" + 1017 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1018 String found = execParser("T.g", grammar, "TParser", "TLexer", 1019 "a", "2", debug); 1020 assertEquals("2\n", found); 1021 } 1022 1023 @Test testSetWithLabel()1024 public void testSetWithLabel() throws Exception { 1025 1026 String grammar = 1027 "grammar T;\n" + 1028 "options { output = AST; } \n" + 1029 "a : x=(INT|ID) -> $x ;\n" + 1030 "INT: '0'..'9'+;\n" + 1031 "ID : 'a'..'z'+;\n" + 1032 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1033 String found = execParser("T.g", grammar, "TParser", "TLexer", 1034 "a", "2", debug); 1035 assertEquals("2\n", found); 1036 } 1037 testRewriteAction()1038 @Test public void testRewriteAction() throws Exception { 1039 String grammar = 1040 "grammar T; \n" + 1041 "options { output = AST; }\n" + 1042 "tokens { FLOAT; }\n" + 1043 "r\n" + 1044 " : INT -> {new CommonTree(new CommonToken(FLOAT,$INT.text+\".0\"))} \n" + 1045 " ; \n" + 1046 "INT : '0'..'9'+; \n" + 1047 "WS: (' ' | '\\n' | '\\t')+ {$channel = HIDDEN;}; \n"; 1048 String found = execParser("T.g", grammar, "TParser", "TLexer", 1049 "r", "25", debug); 1050 assertEquals("25.0\n", found); 1051 } 1052 testOptionalSubruleWithoutRealElements()1053 @Test public void testOptionalSubruleWithoutRealElements() throws Exception { 1054 // copy type *and* modifier even though it's optional 1055 // for each invocation of (...)+ in rewrite 1056 String grammar = 1057 "grammar T;\n" + 1058 "options {output=AST;} \n" + 1059 "tokens {PARMS;} \n" + 1060 "\n" + 1061 "modulo \n" + 1062 " : 'modulo' ID ('(' parms+ ')')? -> ^('modulo' ID ^(PARMS parms+)?) \n" + 1063 " ; \n" + 1064 "parms : '#'|ID; \n" + 1065 "ID : ('a'..'z' | 'A'..'Z')+;\n" + 1066 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1067 String found = execParser("T.g", grammar, "TParser", "TLexer", 1068 "modulo", "modulo abc (x y #)", debug); 1069 assertEquals("(modulo abc (PARMS x y #))\n", found); 1070 } 1071 1072 // C A R D I N A L I T Y I S S U E S 1073 testCardinality()1074 @Test public void testCardinality() throws Exception { 1075 String grammar = 1076 "grammar T;\n" + 1077 "options {output=AST;}\n" + 1078 "tokens {BLOCK;}\n" + 1079 "a : ID ID INT INT INT -> (ID INT)+;\n"+ 1080 "ID : 'a'..'z'+ ;\n" + 1081 "INT : '0'..'9'+; \n" + 1082 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1083 execParser("T.g", grammar, "TParser", "TLexer", 1084 "a", "a b 3 4 5", debug); 1085 String expecting = 1086 "org.antlr.runtime.tree.RewriteCardinalityException: token ID"; 1087 String found = getFirstLineOfException(); 1088 assertEquals(expecting, found); 1089 } 1090 testCardinality2()1091 @Test public void testCardinality2() throws Exception { 1092 String grammar = 1093 "grammar T;\n" + 1094 "options {output=AST;}\n" + 1095 "a : ID+ -> ID ID ID ;\n" + // only 2 input IDs 1096 "op : '+'|'-' ;\n" + 1097 "ID : 'a'..'z'+ ;\n" + 1098 "INT : '0'..'9'+;\n" + 1099 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1100 execParser("T.g", grammar, "TParser", "TLexer", 1101 "a", "a b", debug); 1102 String expecting = 1103 "org.antlr.runtime.tree.RewriteCardinalityException: token ID"; 1104 String found = getFirstLineOfException(); 1105 assertEquals(expecting, found); 1106 } 1107 testCardinality3()1108 @Test public void testCardinality3() throws Exception { 1109 String grammar = 1110 "grammar T;\n" + 1111 "options {output=AST;}\n" + 1112 "a : ID? INT -> ID INT ;\n" + 1113 "op : '+'|'-' ;\n" + 1114 "ID : 'a'..'z'+ ;\n" + 1115 "INT : '0'..'9'+;\n" + 1116 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1117 execParser("T.g", grammar, "TParser", "TLexer", 1118 "a", "3", debug); 1119 String expecting = 1120 "org.antlr.runtime.tree.RewriteEmptyStreamException: token ID"; 1121 String found = getFirstLineOfException(); 1122 assertEquals(expecting, found); 1123 } 1124 testLoopCardinality()1125 @Test public void testLoopCardinality() throws Exception { 1126 String grammar = 1127 "grammar T;\n" + 1128 "options {output=AST;}\n" + 1129 "a : ID? INT -> ID+ INT ;\n" + 1130 "op : '+'|'-' ;\n" + 1131 "ID : 'a'..'z'+ ;\n" + 1132 "INT : '0'..'9'+;\n" + 1133 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1134 execParser("T.g", grammar, "TParser", "TLexer", 1135 "a", "3", debug); 1136 String expecting = 1137 "org.antlr.runtime.tree.RewriteEarlyExitException"; 1138 String found = getFirstLineOfException(); 1139 assertEquals(expecting, found); 1140 } 1141 testWildcard()1142 @Test public void testWildcard() throws Exception { 1143 String grammar = 1144 "grammar T;\n" + 1145 "options {output=AST;}\n" + 1146 "a : ID c=. -> $c;\n" + 1147 "ID : 'a'..'z'+ ;\n" + 1148 "INT : '0'..'9'+;\n" + 1149 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1150 String found = execParser("T.g", grammar, "TParser", "TLexer", 1151 "a", "abc 34", debug); 1152 assertEquals("34\n", found); 1153 } 1154 1155 // E R R O R S 1156 testUnknownRule()1157 @Test public void testUnknownRule() throws Exception { 1158 ErrorQueue equeue = new ErrorQueue(); 1159 ErrorManager.setErrorListener(equeue); 1160 1161 String grammar = 1162 "grammar T;\n" + 1163 "options {output=AST;}\n" + 1164 "a : INT -> ugh ;\n" + 1165 "ID : 'a'..'z'+ ;\n" + 1166 "INT : '0'..'9'+;\n" + 1167 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1168 1169 Grammar g = new Grammar(grammar); 1170 Tool antlr = newTool(); 1171 antlr.setOutputDirectory(null); // write to /dev/null 1172 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1173 g.setCodeGenerator(generator); 1174 generator.genRecognizer(); 1175 1176 int expectedMsgID = ErrorManager.MSG_UNDEFINED_RULE_REF; 1177 Object expectedArg = "ugh"; 1178 Object expectedArg2 = null; 1179 GrammarSemanticsMessage expectedMessage = 1180 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1181 1182 checkError(equeue, expectedMessage); 1183 } 1184 testKnownRuleButNotInLHS()1185 @Test public void testKnownRuleButNotInLHS() throws Exception { 1186 ErrorQueue equeue = new ErrorQueue(); 1187 ErrorManager.setErrorListener(equeue); 1188 1189 String grammar = 1190 "grammar T;\n" + 1191 "options {output=AST;}\n" + 1192 "a : INT -> b ;\n" + 1193 "b : 'b' ;\n" + 1194 "ID : 'a'..'z'+ ;\n" + 1195 "INT : '0'..'9'+;\n" + 1196 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1197 1198 Grammar g = new Grammar(grammar); 1199 Tool antlr = newTool(); 1200 antlr.setOutputDirectory(null); // write to /dev/null 1201 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1202 g.setCodeGenerator(generator); 1203 generator.genRecognizer(); 1204 1205 int expectedMsgID = ErrorManager.MSG_REWRITE_ELEMENT_NOT_PRESENT_ON_LHS; 1206 Object expectedArg = "b"; 1207 Object expectedArg2 = null; 1208 GrammarSemanticsMessage expectedMessage = 1209 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1210 1211 checkError(equeue, expectedMessage); 1212 } 1213 testUnknownToken()1214 @Test public void testUnknownToken() throws Exception { 1215 ErrorQueue equeue = new ErrorQueue(); 1216 ErrorManager.setErrorListener(equeue); 1217 1218 String grammar = 1219 "grammar T;\n" + 1220 "options {output=AST;}\n" + 1221 "a : INT -> ICK ;\n" + 1222 "ID : 'a'..'z'+ ;\n" + 1223 "INT : '0'..'9'+;\n" + 1224 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1225 1226 Grammar g = new Grammar(grammar); 1227 Tool antlr = newTool(); 1228 antlr.setOutputDirectory(null); // write to /dev/null 1229 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1230 g.setCodeGenerator(generator); 1231 generator.genRecognizer(); 1232 1233 int expectedMsgID = ErrorManager.MSG_UNDEFINED_TOKEN_REF_IN_REWRITE; 1234 Object expectedArg = "ICK"; 1235 Object expectedArg2 = null; 1236 GrammarSemanticsMessage expectedMessage = 1237 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1238 1239 checkError(equeue, expectedMessage); 1240 } 1241 testUnknownLabel()1242 @Test public void testUnknownLabel() throws Exception { 1243 ErrorQueue equeue = new ErrorQueue(); 1244 ErrorManager.setErrorListener(equeue); 1245 1246 String grammar = 1247 "grammar T;\n" + 1248 "options {output=AST;}\n" + 1249 "a : INT -> $foo ;\n" + 1250 "ID : 'a'..'z'+ ;\n" + 1251 "INT : '0'..'9'+;\n" + 1252 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1253 1254 Grammar g = new Grammar(grammar); 1255 Tool antlr = newTool(); 1256 antlr.setOutputDirectory(null); // write to /dev/null 1257 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1258 g.setCodeGenerator(generator); 1259 generator.genRecognizer(); 1260 1261 int expectedMsgID = ErrorManager.MSG_UNDEFINED_LABEL_REF_IN_REWRITE; 1262 Object expectedArg = "foo"; 1263 Object expectedArg2 = null; 1264 GrammarSemanticsMessage expectedMessage = 1265 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1266 1267 checkError(equeue, expectedMessage); 1268 } 1269 testUnknownCharLiteralToken()1270 @Test public void testUnknownCharLiteralToken() throws Exception { 1271 ErrorQueue equeue = new ErrorQueue(); 1272 ErrorManager.setErrorListener(equeue); 1273 1274 String grammar = 1275 "grammar T;\n" + 1276 "options {output=AST;}\n" + 1277 "a : INT -> 'a' ;\n" + 1278 "ID : 'a'..'z'+ ;\n" + 1279 "INT : '0'..'9'+;\n" + 1280 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1281 1282 Grammar g = new Grammar(grammar); 1283 Tool antlr = newTool(); 1284 antlr.setOutputDirectory(null); // write to /dev/null 1285 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1286 g.setCodeGenerator(generator); 1287 generator.genRecognizer(); 1288 1289 int expectedMsgID = ErrorManager.MSG_UNDEFINED_TOKEN_REF_IN_REWRITE; 1290 Object expectedArg = "'a'"; 1291 Object expectedArg2 = null; 1292 GrammarSemanticsMessage expectedMessage = 1293 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1294 1295 checkError(equeue, expectedMessage); 1296 } 1297 testUnknownStringLiteralToken()1298 @Test public void testUnknownStringLiteralToken() throws Exception { 1299 ErrorQueue equeue = new ErrorQueue(); 1300 ErrorManager.setErrorListener(equeue); 1301 1302 String grammar = 1303 "grammar T;\n" + 1304 "options {output=AST;}\n" + 1305 "a : INT -> 'foo' ;\n" + 1306 "ID : 'a'..'z'+ ;\n" + 1307 "INT : '0'..'9'+;\n" + 1308 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1309 1310 Grammar g = new Grammar(grammar); 1311 Tool antlr = newTool(); 1312 antlr.setOutputDirectory(null); // write to /dev/null 1313 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1314 g.setCodeGenerator(generator); 1315 generator.genRecognizer(); 1316 1317 int expectedMsgID = ErrorManager.MSG_UNDEFINED_TOKEN_REF_IN_REWRITE; 1318 Object expectedArg = "'foo'"; 1319 Object expectedArg2 = null; 1320 GrammarSemanticsMessage expectedMessage = 1321 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1322 1323 checkError(equeue, expectedMessage); 1324 } 1325 testExtraTokenInSimpleDecl()1326 @Test public void testExtraTokenInSimpleDecl() throws Exception { 1327 String grammar = 1328 "grammar foo;\n" + 1329 "options {output=AST;}\n" + 1330 "tokens {EXPR;}\n" + 1331 "decl : type ID '=' INT ';' -> ^(EXPR type ID INT) ;\n" + 1332 "type : 'int' | 'float' ;\n" + 1333 "ID : 'a'..'z'+ ;\n" + 1334 "INT : '0'..'9'+;\n" + 1335 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1336 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1337 "decl", "int 34 x=1;", debug); 1338 assertEquals("line 1:4 extraneous input '34' expecting ID\n", this.stderrDuringParse); 1339 assertEquals("(EXPR int x 1)\n", found); // tree gets correct x and 1 tokens 1340 } 1341 testMissingIDInSimpleDecl()1342 @Test public void testMissingIDInSimpleDecl() throws Exception { 1343 String grammar = 1344 "grammar foo;\n" + 1345 "options {output=AST;}\n" + 1346 "tokens {EXPR;}\n" + 1347 "decl : type ID '=' INT ';' -> ^(EXPR type ID INT) ;\n" + 1348 "type : 'int' | 'float' ;\n" + 1349 "ID : 'a'..'z'+ ;\n" + 1350 "INT : '0'..'9'+;\n" + 1351 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1352 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1353 "decl", "int =1;", debug); 1354 assertEquals("line 1:4 missing ID at '='\n", this.stderrDuringParse); 1355 assertEquals("(EXPR int <missing ID> 1)\n", found); // tree gets invented ID token 1356 } 1357 testMissingSetInSimpleDecl()1358 @Test public void testMissingSetInSimpleDecl() throws Exception { 1359 String grammar = 1360 "grammar foo;\n" + 1361 "options {output=AST;}\n" + 1362 "tokens {EXPR;}\n" + 1363 "decl : type ID '=' INT ';' -> ^(EXPR type ID INT) ;\n" + 1364 "type : 'int' | 'float' ;\n" + 1365 "ID : 'a'..'z'+ ;\n" + 1366 "INT : '0'..'9'+;\n" + 1367 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1368 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1369 "decl", "x=1;", debug); 1370 assertEquals("line 1:0 mismatched input 'x' expecting set null\n", this.stderrDuringParse); 1371 assertEquals("(EXPR <error: x> x 1)\n", found); // tree gets invented ID token 1372 } 1373 testMissingTokenGivesErrorNode()1374 @Test public void testMissingTokenGivesErrorNode() throws Exception { 1375 String grammar = 1376 "grammar foo;\n" + 1377 "options {output=AST;}\n" + 1378 "a : ID INT -> ID INT ;\n" + 1379 "ID : 'a'..'z'+ ;\n" + 1380 "INT : '0'..'9'+;\n" + 1381 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1382 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1383 "a", "abc", debug); 1384 assertEquals("line 1:3 missing INT at '<EOF>'\n", this.stderrDuringParse); 1385 // doesn't do in-line recovery for sets (yet?) 1386 assertEquals("abc <missing INT>\n", found); 1387 } 1388 testExtraTokenGivesErrorNode()1389 @Test public void testExtraTokenGivesErrorNode() throws Exception { 1390 String grammar = 1391 "grammar foo;\n" + 1392 "options {output=AST;}\n" + 1393 "a : b c -> b c;\n" + 1394 "b : ID -> ID ;\n" + 1395 "c : INT -> INT ;\n" + 1396 "ID : 'a'..'z'+ ;\n" + 1397 "INT : '0'..'9'+;\n" + 1398 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1399 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1400 "a", "abc ick 34", debug); 1401 assertEquals("line 1:4 extraneous input 'ick' expecting INT\n", this.stderrDuringParse); 1402 assertEquals("abc 34\n", found); 1403 } 1404 testMissingFirstTokenGivesErrorNode()1405 @Test public void testMissingFirstTokenGivesErrorNode() throws Exception { 1406 String grammar = 1407 "grammar foo;\n" + 1408 "options {output=AST;}\n" + 1409 "a : ID INT -> ID INT ;\n" + 1410 "ID : 'a'..'z'+ ;\n" + 1411 "INT : '0'..'9'+;\n" + 1412 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1413 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1414 "a", "34", debug); 1415 assertEquals("line 1:0 missing ID at '34'\n", this.stderrDuringParse); 1416 assertEquals("<missing ID> 34\n", found); 1417 } 1418 testMissingFirstTokenGivesErrorNode2()1419 @Test public void testMissingFirstTokenGivesErrorNode2() throws Exception { 1420 String grammar = 1421 "grammar foo;\n" + 1422 "options {output=AST;}\n" + 1423 "a : b c -> b c;\n" + 1424 "b : ID -> ID ;\n" + 1425 "c : INT -> INT ;\n" + 1426 "ID : 'a'..'z'+ ;\n" + 1427 "INT : '0'..'9'+;\n" + 1428 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1429 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1430 "a", "34", debug); 1431 // finds an error at the first token, 34, and re-syncs. 1432 // re-synchronizing does not consume a token because 34 follows 1433 // ref to rule b (start of c). It then matches 34 in c. 1434 assertEquals("line 1:0 missing ID at '34'\n", this.stderrDuringParse); 1435 assertEquals("<missing ID> 34\n", found); 1436 } 1437 testNoViableAltGivesErrorNode()1438 @Test public void testNoViableAltGivesErrorNode() throws Exception { 1439 String grammar = 1440 "grammar foo;\n" + 1441 "options {output=AST;}\n" + 1442 "a : b -> b | c -> c;\n" + 1443 "b : ID -> ID ;\n" + 1444 "c : INT -> INT ;\n" + 1445 "ID : 'a'..'z'+ ;\n" + 1446 "S : '*' ;\n" + 1447 "INT : '0'..'9'+;\n" + 1448 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1449 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1450 "a", "*", debug); 1451 // finds an error at the first token, 34, and re-syncs. 1452 // re-synchronizing does not consume a token because 34 follows 1453 // ref to rule b (start of c). It then matches 34 in c. 1454 assertEquals("line 1:0 no viable alternative at input '*'\n", this.stderrDuringParse); 1455 assertEquals("<unexpected: [@0,0:0='*',<6>,1:0], resync=*>\n", found); 1456 } 1457 testRewriteEmptyRule()1458 @Test public void testRewriteEmptyRule() throws Exception { 1459 String grammar = 1460 "grammar T;\n" + 1461 "options {output=AST;}\n" + 1462 "tokens {IMAGINARY;}\n" + 1463 "a : empty EOF! ;\n" + 1464 "empty : -> IMAGINARY;\n" + 1465 "WS : ' ';\n"; 1466 String result = execParser("T.g", grammar, "TParser", "TLexer", 1467 "a", "", debug); 1468 assertEquals("IMAGINARY\n", result); 1469 assertNull(stderrDuringParse); 1470 } 1471 1472 } 1473