1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 31 import org.antlr.Tool; 32 import org.antlr.analysis.Label; 33 import org.antlr.runtime.CommonTokenStream; 34 import org.antlr.runtime.Token; 35 import org.antlr.runtime.TokenSource; 36 import org.stringtemplate.v4.ST; 37 import org.stringtemplate.v4.STGroup; 38 import org.antlr.tool.ANTLRErrorListener; 39 import org.antlr.tool.ErrorManager; 40 import org.antlr.tool.GrammarSemanticsMessage; 41 import org.antlr.tool.Message; 42 import org.junit.Assert; 43 import org.junit.Before; 44 import org.junit.Rule; 45 import org.junit.rules.TestRule; 46 import org.junit.rules.TestWatcher; 47 import org.junit.runner.Description; 48 49 import javax.tools.*; 50 import java.io.*; 51 import java.lang.reflect.InvocationTargetException; 52 import java.lang.reflect.Method; 53 import java.net.MalformedURLException; 54 import java.net.URL; 55 import java.net.URLClassLoader; 56 import java.util.*; 57 import java.util.logging.Level; 58 import java.util.logging.Logger; 59 60 import static org.junit.Assert.*; 61 62 public abstract class BaseTest { 63 // -J-Dorg.antlr.test.BaseTest.level=FINE 64 private static final Logger LOGGER = Logger.getLogger(BaseTest.class.getName()); 65 66 public static final String newline = System.getProperty("line.separator"); 67 68 public static final String jikes = null;//"/usr/bin/jikes"; 69 public static final String pathSep = System.getProperty("path.separator"); 70 71 public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess")); 72 73 /** 74 * When runnning from Maven, the junit tests are run via the surefire plugin. It sets the 75 * classpath for the test environment into the following property. We need to pick this up 76 * for the junit tests that are going to generate and try to run code. 77 */ 78 public static final String SUREFIRE_CLASSPATH = System.getProperty("surefire.test.class.path", ""); 79 80 /** 81 * Build up the full classpath we need, including the surefire path (if present) 82 */ 83 public static final String CLASSPATH = System.getProperty("java.class.path") + (SUREFIRE_CLASSPATH.equals("") ? "" : pathSep + SUREFIRE_CLASSPATH); 84 85 public String tmpdir = null; 86 87 /** If error during parser execution, store stderr here; can't return 88 * stdout and stderr. This doesn't trap errors from running antlr. 89 */ 90 protected String stderrDuringParse; 91 92 @Rule 93 public final TestRule testWatcher = new TestWatcher() { 94 95 @Override 96 protected void succeeded(Description description) { 97 // remove tmpdir if no error. 98 eraseTempDir(); 99 } 100 101 }; 102 103 @Before setUp()104 public void setUp() throws Exception { 105 // new output dir for each test 106 tmpdir = new File(System.getProperty("java.io.tmpdir"), 107 "antlr-"+getClass().getName()+"-"+ 108 System.currentTimeMillis()).getAbsolutePath(); 109 ErrorManager.resetErrorState(); 110 STGroup.defaultGroup = new STGroup(); 111 } 112 newTool(String[] args)113 protected Tool newTool(String[] args) { 114 Tool tool = new Tool(args); 115 tool.setOutputDirectory(tmpdir); 116 return tool; 117 } 118 newTool()119 protected Tool newTool() { 120 Tool tool = new Tool(); 121 tool.setOutputDirectory(tmpdir); 122 return tool; 123 } 124 compile(String fileName)125 protected boolean compile(String fileName) { 126 String classpathOption = "-classpath"; 127 128 String[] args = new String[] { 129 "javac", "-d", tmpdir, 130 classpathOption, tmpdir+pathSep+CLASSPATH, 131 tmpdir+"/"+fileName 132 }; 133 String cmdLine = "javac" +" -d "+tmpdir+" "+classpathOption+" "+tmpdir+pathSep+CLASSPATH+" "+fileName; 134 //System.out.println("compile: "+cmdLine); 135 136 137 File f = new File(tmpdir, fileName); 138 JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); 139 140 StandardJavaFileManager fileManager = 141 compiler.getStandardFileManager(null, null, null); 142 143 Iterable<? extends JavaFileObject> compilationUnits = 144 fileManager.getJavaFileObjectsFromFiles(Arrays.asList(f)); 145 146 Iterable<String> compileOptions = 147 Arrays.asList(new String[]{"-d", tmpdir, "-cp", tmpdir+pathSep+CLASSPATH} ); 148 149 JavaCompiler.CompilationTask task = 150 compiler.getTask(null, fileManager, null, compileOptions, null, 151 compilationUnits); 152 boolean ok = task.call(); 153 154 try { 155 fileManager.close(); 156 } 157 catch (IOException ioe) { 158 ioe.printStackTrace(System.err); 159 } 160 return ok; 161 } 162 163 /** Return true if all is ok, no errors */ antlr(String fileName, String grammarFileName, String grammarStr, boolean debug)164 protected boolean antlr(String fileName, String grammarFileName, String grammarStr, boolean debug) { 165 boolean allIsWell = true; 166 mkdir(tmpdir); 167 writeFile(tmpdir, fileName, grammarStr); 168 try { 169 final List<String> options = new ArrayList<String>(); 170 if ( debug ) { 171 options.add("-debug"); 172 } 173 options.add("-o"); 174 options.add(tmpdir); 175 options.add("-lib"); 176 options.add(tmpdir); 177 options.add(new File(tmpdir,grammarFileName).toString()); 178 final String[] optionsA = new String[options.size()]; 179 options.toArray(optionsA); 180 /* 181 final ErrorQueue equeue = new ErrorQueue(); 182 ErrorManager.setErrorListener(equeue); 183 */ 184 Tool antlr = newTool(optionsA); 185 antlr.process(); 186 ANTLRErrorListener listener = ErrorManager.getErrorListener(); 187 if ( listener instanceof ErrorQueue ) { 188 ErrorQueue equeue = (ErrorQueue)listener; 189 if ( equeue.errors.size()>0 ) { 190 allIsWell = false; 191 System.err.println("antlr reports errors from "+options); 192 for (int i = 0; i < equeue.errors.size(); i++) { 193 Message msg = equeue.errors.get(i); 194 System.err.println(msg); 195 } 196 System.out.println("!!!\ngrammar:"); 197 System.out.println(grammarStr); 198 System.out.println("###"); 199 } 200 } 201 } 202 catch (Exception e) { 203 allIsWell = false; 204 System.err.println("problems building grammar: "+e); 205 e.printStackTrace(System.err); 206 } 207 return allIsWell; 208 } 209 execLexer(String grammarFileName, String grammarStr, String lexerName, String input, boolean debug)210 protected String execLexer(String grammarFileName, 211 String grammarStr, 212 String lexerName, 213 String input, 214 boolean debug) 215 { 216 boolean compiled = rawGenerateAndBuildRecognizer(grammarFileName, 217 grammarStr, 218 null, 219 lexerName, 220 debug); 221 Assert.assertTrue(compiled); 222 223 writeFile(tmpdir, "input", input); 224 return rawExecRecognizer(null, 225 null, 226 lexerName, 227 null, 228 null, 229 false, 230 false, 231 false, 232 debug); 233 } 234 execParser(String grammarFileName, String grammarStr, String parserName, String lexerName, String startRuleName, String input, boolean debug)235 protected String execParser(String grammarFileName, 236 String grammarStr, 237 String parserName, 238 String lexerName, 239 String startRuleName, 240 String input, boolean debug) 241 { 242 boolean compiled = rawGenerateAndBuildRecognizer(grammarFileName, 243 grammarStr, 244 parserName, 245 lexerName, 246 debug); 247 Assert.assertTrue(compiled); 248 249 writeFile(tmpdir, "input", input); 250 boolean parserBuildsTrees = 251 grammarStr.indexOf("output=AST")>=0 || 252 grammarStr.indexOf("output = AST")>=0; 253 boolean parserBuildsTemplate = 254 grammarStr.indexOf("output=template")>=0 || 255 grammarStr.indexOf("output = template")>=0; 256 return rawExecRecognizer(parserName, 257 null, 258 lexerName, 259 startRuleName, 260 null, 261 parserBuildsTrees, 262 parserBuildsTemplate, 263 false, 264 debug); 265 } 266 execTreeParser(String parserGrammarFileName, String parserGrammarStr, String parserName, String treeParserGrammarFileName, String treeParserGrammarStr, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, String input)267 protected String execTreeParser(String parserGrammarFileName, 268 String parserGrammarStr, 269 String parserName, 270 String treeParserGrammarFileName, 271 String treeParserGrammarStr, 272 String treeParserName, 273 String lexerName, 274 String parserStartRuleName, 275 String treeParserStartRuleName, 276 String input) 277 { 278 return execTreeParser(parserGrammarFileName, 279 parserGrammarStr, 280 parserName, 281 treeParserGrammarFileName, 282 treeParserGrammarStr, 283 treeParserName, 284 lexerName, 285 parserStartRuleName, 286 treeParserStartRuleName, 287 input, 288 false); 289 } 290 execTreeParser(String parserGrammarFileName, String parserGrammarStr, String parserName, String treeParserGrammarFileName, String treeParserGrammarStr, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, String input, boolean debug)291 protected String execTreeParser(String parserGrammarFileName, 292 String parserGrammarStr, 293 String parserName, 294 String treeParserGrammarFileName, 295 String treeParserGrammarStr, 296 String treeParserName, 297 String lexerName, 298 String parserStartRuleName, 299 String treeParserStartRuleName, 300 String input, 301 boolean debug) 302 { 303 // build the parser 304 boolean compiled = rawGenerateAndBuildRecognizer(parserGrammarFileName, 305 parserGrammarStr, 306 parserName, 307 lexerName, 308 debug); 309 Assert.assertTrue(compiled); 310 311 // build the tree parser 312 compiled = rawGenerateAndBuildRecognizer(treeParserGrammarFileName, 313 treeParserGrammarStr, 314 treeParserName, 315 lexerName, 316 debug); 317 Assert.assertTrue(compiled); 318 319 writeFile(tmpdir, "input", input); 320 321 boolean parserBuildsTrees = 322 parserGrammarStr.indexOf("output=AST")>=0 || 323 parserGrammarStr.indexOf("output = AST")>=0; 324 boolean treeParserBuildsTrees = 325 treeParserGrammarStr.indexOf("output=AST")>=0 || 326 treeParserGrammarStr.indexOf("output = AST")>=0; 327 boolean parserBuildsTemplate = 328 parserGrammarStr.indexOf("output=template")>=0 || 329 parserGrammarStr.indexOf("output = template")>=0; 330 331 return rawExecRecognizer(parserName, 332 treeParserName, 333 lexerName, 334 parserStartRuleName, 335 treeParserStartRuleName, 336 parserBuildsTrees, 337 parserBuildsTemplate, 338 treeParserBuildsTrees, 339 debug); 340 } 341 342 /** Return true if all is well */ rawGenerateAndBuildRecognizer(String grammarFileName, String grammarStr, String parserName, String lexerName, boolean debug)343 protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, 344 String grammarStr, 345 String parserName, 346 String lexerName, 347 boolean debug) 348 { 349 //System.out.println(grammarStr); 350 boolean allIsWell = 351 antlr(grammarFileName, grammarFileName, grammarStr, debug); 352 if (!allIsWell) { 353 return false; 354 } 355 356 if ( lexerName!=null ) { 357 boolean ok; 358 if ( parserName!=null ) { 359 ok = compile(parserName+".java"); 360 if ( !ok ) { allIsWell = false; } 361 } 362 ok = compile(lexerName+".java"); 363 if ( !ok ) { allIsWell = false; } 364 } 365 else { 366 boolean ok = compile(parserName+".java"); 367 if ( !ok ) { allIsWell = false; } 368 } 369 return allIsWell; 370 } 371 rawExecRecognizer(String parserName, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, boolean parserBuildsTrees, boolean parserBuildsTemplate, boolean treeParserBuildsTrees, boolean debug)372 protected String rawExecRecognizer(String parserName, 373 String treeParserName, 374 String lexerName, 375 String parserStartRuleName, 376 String treeParserStartRuleName, 377 boolean parserBuildsTrees, 378 boolean parserBuildsTemplate, 379 boolean treeParserBuildsTrees, 380 boolean debug) 381 { 382 this.stderrDuringParse = null; 383 writeRecognizerAndCompile(parserName, treeParserName, lexerName, parserStartRuleName, treeParserStartRuleName, parserBuildsTrees, parserBuildsTemplate, treeParserBuildsTrees, debug); 384 385 return execRecognizer(); 386 } 387 execRecognizer()388 public String execRecognizer() { 389 return execClass("Test"); 390 } 391 execClass(String className)392 public String execClass(String className) { 393 if (TEST_IN_SAME_PROCESS) { 394 try { 395 ClassLoader loader = new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, ClassLoader.getSystemClassLoader()); 396 final Class<?> mainClass = (Class<?>)loader.loadClass(className); 397 final Method mainMethod = mainClass.getDeclaredMethod("main", String[].class); 398 PipedInputStream stdoutIn = new PipedInputStream(); 399 PipedInputStream stderrIn = new PipedInputStream(); 400 PipedOutputStream stdoutOut = new PipedOutputStream(stdoutIn); 401 PipedOutputStream stderrOut = new PipedOutputStream(stderrIn); 402 String inputFile = new File(tmpdir, "input").getAbsolutePath(); 403 StreamVacuum stdoutVacuum = new StreamVacuum(stdoutIn, inputFile); 404 StreamVacuum stderrVacuum = new StreamVacuum(stderrIn, inputFile); 405 406 PrintStream originalOut = System.out; 407 System.setOut(new PrintStream(stdoutOut)); 408 try { 409 PrintStream originalErr = System.err; 410 try { 411 System.setErr(new PrintStream(stderrOut)); 412 stdoutVacuum.start(); 413 stderrVacuum.start(); 414 mainMethod.invoke(null, (Object)new String[] { inputFile }); 415 } 416 finally { 417 System.setErr(originalErr); 418 } 419 } 420 finally { 421 System.setOut(originalOut); 422 } 423 424 stdoutOut.close(); 425 stderrOut.close(); 426 stdoutVacuum.join(); 427 stderrVacuum.join(); 428 String output = stdoutVacuum.toString(); 429 if ( stderrVacuum.toString().length()>0 ) { 430 this.stderrDuringParse = stderrVacuum.toString(); 431 System.err.println("exec stderrVacuum: "+ stderrVacuum); 432 } 433 return output; 434 } catch (MalformedURLException ex) { 435 LOGGER.log(Level.SEVERE, null, ex); 436 } catch (IOException ex) { 437 LOGGER.log(Level.SEVERE, null, ex); 438 } catch (InterruptedException ex) { 439 LOGGER.log(Level.SEVERE, null, ex); 440 } catch (IllegalAccessException ex) { 441 LOGGER.log(Level.SEVERE, null, ex); 442 } catch (IllegalArgumentException ex) { 443 LOGGER.log(Level.SEVERE, null, ex); 444 } catch (InvocationTargetException ex) { 445 LOGGER.log(Level.SEVERE, null, ex); 446 } catch (NoSuchMethodException ex) { 447 LOGGER.log(Level.SEVERE, null, ex); 448 } catch (SecurityException ex) { 449 LOGGER.log(Level.SEVERE, null, ex); 450 } catch (ClassNotFoundException ex) { 451 LOGGER.log(Level.SEVERE, null, ex); 452 } 453 } 454 455 try { 456 String inputFile = new File(tmpdir, "input").getAbsolutePath(); 457 String[] args = new String[] { 458 "java", "-classpath", tmpdir+pathSep+CLASSPATH, 459 className, inputFile 460 }; 461 //String cmdLine = "java -classpath "+CLASSPATH+pathSep+tmpdir+" Test " + new File(tmpdir, "input").getAbsolutePath(); 462 //System.out.println("execParser: "+cmdLine); 463 Process process = 464 Runtime.getRuntime().exec(args, null, new File(tmpdir)); 465 StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream(), inputFile); 466 StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream(), inputFile); 467 stdoutVacuum.start(); 468 stderrVacuum.start(); 469 process.waitFor(); 470 stdoutVacuum.join(); 471 stderrVacuum.join(); 472 String output; 473 output = stdoutVacuum.toString(); 474 if ( stderrVacuum.toString().length()>0 ) { 475 this.stderrDuringParse = stderrVacuum.toString(); 476 System.err.println("exec stderrVacuum: "+ stderrVacuum); 477 } 478 return output; 479 } 480 catch (Exception e) { 481 System.err.println("can't exec recognizer"); 482 e.printStackTrace(System.err); 483 } 484 return null; 485 } 486 writeRecognizerAndCompile(String parserName, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, boolean parserBuildsTrees, boolean parserBuildsTemplate, boolean treeParserBuildsTrees, boolean debug)487 public void writeRecognizerAndCompile(String parserName, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, boolean parserBuildsTrees, boolean parserBuildsTemplate, boolean treeParserBuildsTrees, boolean debug) { 488 if ( treeParserBuildsTrees && parserBuildsTrees ) { 489 writeTreeAndTreeTestFile(parserName, 490 treeParserName, 491 lexerName, 492 parserStartRuleName, 493 treeParserStartRuleName, 494 debug); 495 } 496 else if ( parserBuildsTrees ) { 497 writeTreeTestFile(parserName, 498 treeParserName, 499 lexerName, 500 parserStartRuleName, 501 treeParserStartRuleName, 502 debug); 503 } 504 else if ( parserBuildsTemplate ) { 505 writeTemplateTestFile(parserName, 506 lexerName, 507 parserStartRuleName, 508 debug); 509 } 510 else if ( parserName==null ) { 511 writeLexerTestFile(lexerName, debug); 512 } 513 else { 514 writeTestFile(parserName, 515 lexerName, 516 parserStartRuleName, 517 debug); 518 } 519 520 compile("Test.java"); 521 } 522 checkGrammarSemanticsError(ErrorQueue equeue, GrammarSemanticsMessage expectedMessage)523 protected void checkGrammarSemanticsError(ErrorQueue equeue, 524 GrammarSemanticsMessage expectedMessage) 525 throws Exception 526 { 527 /* 528 System.out.println(equeue.infos); 529 System.out.println(equeue.warnings); 530 System.out.println(equeue.errors); 531 assertTrue("number of errors mismatch", n, equeue.errors.size()); 532 */ 533 Message foundMsg = null; 534 for (int i = 0; i < equeue.errors.size(); i++) { 535 Message m = equeue.errors.get(i); 536 if (m.msgID==expectedMessage.msgID ) { 537 foundMsg = m; 538 } 539 } 540 assertNotNull("no error; "+expectedMessage.msgID+" expected", foundMsg); 541 assertTrue("error is not a GrammarSemanticsMessage", 542 foundMsg instanceof GrammarSemanticsMessage); 543 assertEquals(expectedMessage.arg, foundMsg.arg); 544 if ( equeue.size()!=1 ) { 545 System.err.println(equeue); 546 } 547 } 548 checkGrammarSemanticsWarning(ErrorQueue equeue, GrammarSemanticsMessage expectedMessage)549 protected void checkGrammarSemanticsWarning(ErrorQueue equeue, 550 GrammarSemanticsMessage expectedMessage) 551 throws Exception 552 { 553 Message foundMsg = null; 554 for (int i = 0; i < equeue.warnings.size(); i++) { 555 Message m = equeue.warnings.get(i); 556 if (m.msgID==expectedMessage.msgID ) { 557 foundMsg = m; 558 } 559 } 560 assertNotNull("no error; "+expectedMessage.msgID+" expected", foundMsg); 561 assertTrue("error is not a GrammarSemanticsMessage", 562 foundMsg instanceof GrammarSemanticsMessage); 563 assertEquals(expectedMessage.arg, foundMsg.arg); 564 } 565 checkError(ErrorQueue equeue, Message expectedMessage)566 protected void checkError(ErrorQueue equeue, 567 Message expectedMessage) 568 throws Exception 569 { 570 //System.out.println("errors="+equeue); 571 Message foundMsg = null; 572 for (int i = 0; i < equeue.errors.size(); i++) { 573 Message m = equeue.errors.get(i); 574 if (m.msgID==expectedMessage.msgID ) { 575 foundMsg = m; 576 } 577 } 578 assertTrue("no error; "+expectedMessage.msgID+" expected", equeue.errors.size()>0); 579 assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1); 580 assertNotNull("couldn't find expected error: "+expectedMessage.msgID, foundMsg); 581 /* 582 assertTrue("error is not a GrammarSemanticsMessage", 583 foundMsg instanceof GrammarSemanticsMessage); 584 */ 585 assertEquals(expectedMessage.arg, foundMsg.arg); 586 assertEquals(expectedMessage.arg2, foundMsg.arg2); 587 ErrorManager.resetErrorState(); // wack errors for next test 588 } 589 590 public static class StreamVacuum implements Runnable { 591 StringBuffer buf = new StringBuffer(); 592 BufferedReader in; 593 Thread sucker; 594 String inputFile; StreamVacuum(InputStream in, String inputFile)595 public StreamVacuum(InputStream in, String inputFile) { 596 this.in = new BufferedReader( new InputStreamReader(in) ); 597 this.inputFile = inputFile; 598 } start()599 public void start() { 600 sucker = new Thread(this); 601 sucker.start(); 602 } 603 @Override run()604 public void run() { 605 try { 606 String line = in.readLine(); 607 while (line!=null) { 608 if (line.startsWith(inputFile)) 609 line = line.substring(inputFile.length()+1); 610 buf.append(line); 611 buf.append('\n'); 612 line = in.readLine(); 613 } 614 } 615 catch (IOException ioe) { 616 System.err.println("can't read output from process"); 617 } 618 } 619 /** wait for the thread to finish */ join()620 public void join() throws InterruptedException { 621 sucker.join(); 622 } 623 @Override toString()624 public String toString() { 625 return buf.toString(); 626 } 627 } 628 629 public static class FilteringTokenStream extends CommonTokenStream { FilteringTokenStream(TokenSource src)630 public FilteringTokenStream(TokenSource src) { super(src); } 631 Set<Integer> hide = new HashSet<Integer>(); 632 @Override sync(int i)633 protected void sync(int i) { 634 super.sync(i); 635 if ( hide.contains(get(i).getType()) ) get(i).setChannel(Token.HIDDEN_CHANNEL); 636 } setTokenTypeChannel(int ttype, int channel)637 public void setTokenTypeChannel(int ttype, int channel) { 638 hide.add(ttype); 639 } 640 } 641 writeFile(String dir, String fileName, String content)642 protected void writeFile(String dir, String fileName, String content) { 643 try { 644 File f = new File(dir, fileName); 645 FileWriter w = new FileWriter(f); 646 BufferedWriter bw = new BufferedWriter(w); 647 bw.write(content); 648 bw.close(); 649 w.close(); 650 } 651 catch (IOException ioe) { 652 System.err.println("can't write file"); 653 ioe.printStackTrace(System.err); 654 } 655 } 656 mkdir(String dir)657 protected void mkdir(String dir) { 658 File f = new File(dir); 659 f.mkdirs(); 660 } 661 writeTestFile(String parserName, String lexerName, String parserStartRuleName, boolean debug)662 protected void writeTestFile(String parserName, 663 String lexerName, 664 String parserStartRuleName, 665 boolean debug) 666 { 667 ST outputFileST = new ST( 668 "import org.antlr.runtime.*;\n" + 669 "import org.antlr.runtime.tree.*;\n" + 670 "import org.antlr.runtime.debug.*;\n" + 671 "\n" + 672 "class Profiler2 extends Profiler {\n" + 673 " public void terminate() { ; }\n" + 674 "}\n"+ 675 "public class Test {\n" + 676 " public static void main(String[] args) throws Exception {\n" + 677 " CharStream input = new ANTLRFileStream(args[0]);\n" + 678 " <lexerName> lex = new <lexerName>(input);\n" + 679 " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + 680 " <createParser>\n"+ 681 " parser.<parserStartRuleName>();\n" + 682 " }\n" + 683 "}" 684 ); 685 ST createParserST = 686 new ST( 687 " Profiler2 profiler = new Profiler2();\n"+ 688 " <parserName> parser = new <parserName>(tokens,profiler);\n" + 689 " profiler.setParser(parser);\n"); 690 if ( !debug ) { 691 createParserST = 692 new ST( 693 " <parserName> parser = new <parserName>(tokens);\n"); 694 } 695 outputFileST.add("createParser", createParserST); 696 outputFileST.add("parserName", parserName); 697 outputFileST.add("lexerName", lexerName); 698 outputFileST.add("parserStartRuleName", parserStartRuleName); 699 writeFile(tmpdir, "Test.java", outputFileST.render()); 700 } 701 writeLexerTestFile(String lexerName, boolean debug)702 protected void writeLexerTestFile(String lexerName, boolean debug) { 703 ST outputFileST = new ST( 704 "import org.antlr.runtime.*;\n" + 705 "import org.antlr.runtime.tree.*;\n" + 706 "import org.antlr.runtime.debug.*;\n" + 707 "\n" + 708 "class Profiler2 extends Profiler {\n" + 709 " public void terminate() { ; }\n" + 710 "}\n"+ 711 "public class Test {\n" + 712 " public static void main(String[] args) throws Exception {\n" + 713 " CharStream input = new ANTLRFileStream(args[0]);\n" + 714 " <lexerName> lex = new <lexerName>(input);\n" + 715 " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + 716 " System.out.println(tokens);\n" + 717 " }\n" + 718 "}" 719 ); 720 outputFileST.add("lexerName", lexerName); 721 writeFile(tmpdir, "Test.java", outputFileST.render()); 722 } 723 writeTreeTestFile(String parserName, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, boolean debug)724 protected void writeTreeTestFile(String parserName, 725 String treeParserName, 726 String lexerName, 727 String parserStartRuleName, 728 String treeParserStartRuleName, 729 boolean debug) 730 { 731 ST outputFileST = new ST( 732 "import org.antlr.runtime.*;\n" + 733 "import org.antlr.runtime.tree.*;\n" + 734 "import org.antlr.runtime.debug.*;\n" + 735 "\n" + 736 "class Profiler2 extends Profiler {\n" + 737 " public void terminate() { ; }\n" + 738 "}\n"+ 739 "public class Test {\n" + 740 " public static void main(String[] args) throws Exception {\n" + 741 " CharStream input = new ANTLRFileStream(args[0]);\n" + 742 " <lexerName> lex = new <lexerName>(input);\n" + 743 " TokenRewriteStream tokens = new TokenRewriteStream(lex);\n" + 744 " <createParser>\n"+ 745 " <parserName>.<parserStartRuleName>_return r = parser.<parserStartRuleName>();\n" + 746 " <if(!treeParserStartRuleName)>\n" + 747 " if ( r.tree!=null ) {\n" + 748 " System.out.println(((Tree)r.tree).toStringTree());\n" + 749 " ((CommonTree)r.tree).sanityCheckParentAndChildIndexes();\n" + 750 " }\n" + 751 " <else>\n" + 752 " CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree);\n" + 753 " nodes.setTokenStream(tokens);\n" + 754 " <treeParserName> walker = new <treeParserName>(nodes);\n" + 755 " walker.<treeParserStartRuleName>();\n" + 756 " <endif>\n" + 757 " }\n" + 758 "}" 759 ); 760 ST createParserST = 761 new ST( 762 " Profiler2 profiler = new Profiler2();\n"+ 763 " <parserName> parser = new <parserName>(tokens,profiler);\n" + 764 " profiler.setParser(parser);\n"); 765 if ( !debug ) { 766 createParserST = 767 new ST( 768 " <parserName> parser = new <parserName>(tokens);\n"); 769 } 770 outputFileST.add("createParser", createParserST); 771 outputFileST.add("parserName", parserName); 772 outputFileST.add("treeParserName", treeParserName); 773 outputFileST.add("lexerName", lexerName); 774 outputFileST.add("parserStartRuleName", parserStartRuleName); 775 outputFileST.add("treeParserStartRuleName", treeParserStartRuleName); 776 writeFile(tmpdir, "Test.java", outputFileST.render()); 777 } 778 779 /** Parser creates trees and so does the tree parser */ writeTreeAndTreeTestFile(String parserName, String treeParserName, String lexerName, String parserStartRuleName, String treeParserStartRuleName, boolean debug)780 protected void writeTreeAndTreeTestFile(String parserName, 781 String treeParserName, 782 String lexerName, 783 String parserStartRuleName, 784 String treeParserStartRuleName, 785 boolean debug) 786 { 787 ST outputFileST = new ST( 788 "import org.antlr.runtime.*;\n" + 789 "import org.antlr.runtime.tree.*;\n" + 790 "import org.antlr.runtime.debug.*;\n" + 791 "\n" + 792 "class Profiler2 extends Profiler {\n" + 793 " public void terminate() { ; }\n" + 794 "}\n"+ 795 "public class Test {\n" + 796 " public static void main(String[] args) throws Exception {\n" + 797 " CharStream input = new ANTLRFileStream(args[0]);\n" + 798 " <lexerName> lex = new <lexerName>(input);\n" + 799 " TokenRewriteStream tokens = new TokenRewriteStream(lex);\n" + 800 " <createParser>\n"+ 801 " <parserName>.<parserStartRuleName>_return r = parser.<parserStartRuleName>();\n" + 802 " ((CommonTree)r.tree).sanityCheckParentAndChildIndexes();\n" + 803 " CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree);\n" + 804 " nodes.setTokenStream(tokens);\n" + 805 " <treeParserName> walker = new <treeParserName>(nodes);\n" + 806 " <treeParserName>.<treeParserStartRuleName>_return r2 = walker.<treeParserStartRuleName>();\n" + 807 " CommonTree rt = ((CommonTree)r2.tree);\n" + 808 " if ( rt!=null ) System.out.println(((CommonTree)r2.tree).toStringTree());\n" + 809 " }\n" + 810 "}" 811 ); 812 ST createParserST = 813 new ST( 814 " Profiler2 profiler = new Profiler2();\n"+ 815 " <parserName> parser = new <parserName>(tokens,profiler);\n" + 816 " profiler.setParser(parser);\n"); 817 if ( !debug ) { 818 createParserST = 819 new ST( 820 " <parserName> parser = new <parserName>(tokens);\n"); 821 } 822 outputFileST.add("createParser", createParserST); 823 outputFileST.add("parserName", parserName); 824 outputFileST.add("treeParserName", treeParserName); 825 outputFileST.add("lexerName", lexerName); 826 outputFileST.add("parserStartRuleName", parserStartRuleName); 827 outputFileST.add("treeParserStartRuleName", treeParserStartRuleName); 828 writeFile(tmpdir, "Test.java", outputFileST.render()); 829 } 830 writeTemplateTestFile(String parserName, String lexerName, String parserStartRuleName, boolean debug)831 protected void writeTemplateTestFile(String parserName, 832 String lexerName, 833 String parserStartRuleName, 834 boolean debug) 835 { 836 ST outputFileST = new ST( 837 "import org.antlr.runtime.*;\n" + 838 "import org.antlr.stringtemplate.*;\n" + 839 "import org.antlr.stringtemplate.language.*;\n" + 840 "import org.antlr.runtime.debug.*;\n" + 841 "import java.io.*;\n" + 842 "\n" + 843 "class Profiler2 extends Profiler {\n" + 844 " public void terminate() { ; }\n" + 845 "}\n"+ 846 "public class Test {\n" + 847 " static String templates = \"group T; foo(x,y) ::= \\\"\\<x> \\<y>\\\"\";\n" + 848 " static StringTemplateGroup group ="+ 849 " new StringTemplateGroup(new StringReader(templates)," + 850 " AngleBracketTemplateLexer.class);"+ 851 " public static void main(String[] args) throws Exception {\n" + 852 " CharStream input = new ANTLRFileStream(args[0]);\n" + 853 " <lexerName> lex = new <lexerName>(input);\n" + 854 " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + 855 " <createParser>\n"+ 856 " parser.setTemplateLib(group);\n"+ 857 " <parserName>.<parserStartRuleName>_return r = parser.<parserStartRuleName>();\n" + 858 " if ( r.st!=null )\n" + 859 " System.out.print(r.st.toString());\n" + 860 " else\n" + 861 " System.out.print(\"\");\n" + 862 " }\n" + 863 "}" 864 ); 865 ST createParserST = 866 new ST( 867 " Profiler2 profiler = new Profiler2();\n"+ 868 " <parserName> parser = new <parserName>(tokens,profiler);\n" + 869 " profiler.setParser(parser);\n"); 870 if ( !debug ) { 871 createParserST = 872 new ST( 873 " <parserName> parser = new <parserName>(tokens);\n"); 874 } 875 outputFileST.add("createParser", createParserST); 876 outputFileST.add("parserName", parserName); 877 outputFileST.add("lexerName", lexerName); 878 outputFileST.add("parserStartRuleName", parserStartRuleName); 879 writeFile(tmpdir, "Test.java", outputFileST.render()); 880 } 881 eraseFiles(final String filesEndingWith)882 protected void eraseFiles(final String filesEndingWith) { 883 File tmpdirF = new File(tmpdir); 884 String[] files = tmpdirF.list(); 885 for(int i = 0; files!=null && i < files.length; i++) { 886 if ( files[i].endsWith(filesEndingWith) ) { 887 new File(tmpdir+"/"+files[i]).delete(); 888 } 889 } 890 } 891 eraseFiles()892 protected void eraseFiles() { 893 File tmpdirF = new File(tmpdir); 894 String[] files = tmpdirF.list(); 895 for(int i = 0; files!=null && i < files.length; i++) { 896 new File(tmpdir+"/"+files[i]).delete(); 897 } 898 } 899 eraseTempDir()900 protected void eraseTempDir() { 901 File tmpdirF = new File(tmpdir); 902 if ( tmpdirF.exists() ) { 903 eraseFiles(); 904 tmpdirF.delete(); 905 } 906 } 907 getFirstLineOfException()908 public String getFirstLineOfException() { 909 if ( this.stderrDuringParse ==null ) { 910 return null; 911 } 912 String[] lines = this.stderrDuringParse.split("\n"); 913 String prefix="Exception in thread \"main\" "; 914 return lines[0].substring(prefix.length(),lines[0].length()); 915 } 916 realElements(List<T> elements)917 public <T> List<T> realElements(List<T> elements) { 918 List<T> n = new ArrayList<T>(); 919 for (int i = Label.NUM_FAUX_LABELS+Label.MIN_TOKEN_TYPE - 1; i < elements.size(); i++) { 920 T o = elements.get(i); 921 if ( o!=null ) { 922 n.add(o); 923 } 924 } 925 return n; 926 } 927 realElements(Map<String, Integer> elements)928 public List<String> realElements(Map<String, Integer> elements) { 929 List<String> n = new ArrayList<String>(); 930 for (Map.Entry<String, Integer> entry : elements.entrySet()) { 931 String tokenID = entry.getKey(); 932 if ( entry.getValue() >= Label.MIN_TOKEN_TYPE ) { 933 n.add(tokenID+"="+entry.getValue()); 934 } 935 } 936 Collections.sort(n); 937 return n; 938 } 939 sortLinesInString(String s)940 public String sortLinesInString(String s) { 941 String lines[] = s.split("\n"); 942 Arrays.sort(lines); 943 List<String> linesL = Arrays.asList(lines); 944 StringBuilder buf = new StringBuilder(); 945 for (String l : linesL) { 946 buf.append(l); 947 buf.append('\n'); 948 } 949 return buf.toString(); 950 } 951 952 /** 953 * When looking at a result set that consists of a Map/HashTable 954 * we cannot rely on the output order, as the hashing algorithm or other aspects 955 * of the implementation may be different on differnt JDKs or platforms. Hence 956 * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a 957 * bit of a hack, but guarantees that we get the same order on all systems. We assume that 958 * the keys are strings. 959 * 960 * @param m The Map that contains keys we wish to return in sorted order 961 * @return A string that represents all the keys in sorted order. 962 */ sortMapToString(Map<K, V> m)963 public <K, V> String sortMapToString(Map<K, V> m) { 964 965 System.out.println("Map toString looks like: " + m.toString()); 966 // Pass in crap, and get nothing back 967 // 968 if (m == null) { 969 return null; 970 } 971 972 // Sort the keys in the Map 973 // 974 TreeMap<K, V> nset = new TreeMap<K, V>(m); 975 976 System.out.println("Tree map looks like: " + nset.toString()); 977 return nset.toString(); 978 } 979 } 980