1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 import org.antlr.runtime.ANTLRStringStream; 31 import org.antlr.runtime.CharStream; 32 import org.antlr.runtime.CommonTokenStream; 33 import org.antlr.runtime.Token; 34 import org.antlr.tool.Grammar; 35 import org.antlr.tool.Interpreter; 36 import org.junit.Test; 37 38 import static org.junit.Assert.*; 39 40 public class TestInterpretedLexing extends BaseTest { 41 42 /* 43 static class Tracer implements ANTLRDebugInterface { 44 Grammar g; 45 public DebugActions(Grammar g) { 46 this.g = g; 47 } 48 public void enterRule(String ruleName) { 49 System.out.println("enterRule("+ruleName+")"); 50 } 51 52 public void exitRule(String ruleName) { 53 System.out.println("exitRule("+ruleName+")"); 54 } 55 56 public void matchElement(int type) { 57 System.out.println("matchElement("+g.getTokenName(type)+")"); 58 } 59 60 public void mismatchedElement(MismatchedTokenException e) { 61 System.out.println(e); 62 e.printStackTrace(System.out); 63 } 64 65 public void mismatchedSet(MismatchedSetException e) { 66 System.out.println(e); 67 e.printStackTrace(System.out); 68 } 69 70 public void noViableAlt(NoViableAltException e) { 71 System.out.println(e); 72 e.printStackTrace(System.out); 73 } 74 } 75 */ 76 77 /** Public default constructor used by TestRig */ TestInterpretedLexing()78 public TestInterpretedLexing() { 79 } 80 testSimpleAltCharTest()81 @Test public void testSimpleAltCharTest() throws Exception { 82 Grammar g = new Grammar( 83 "lexer grammar t;\n"+ 84 "A : 'a' | 'b' | 'c';"); 85 final int Atype = g.getTokenType("A"); 86 Interpreter engine = new Interpreter(g, new ANTLRStringStream("a")); 87 engine = new Interpreter(g, new ANTLRStringStream("b")); 88 Token result = engine.scan("A"); 89 assertEquals(result.getType(), Atype); 90 engine = new Interpreter(g, new ANTLRStringStream("c")); 91 result = engine.scan("A"); 92 assertEquals(result.getType(), Atype); 93 } 94 testSingleRuleRef()95 @Test public void testSingleRuleRef() throws Exception { 96 Grammar g = new Grammar( 97 "lexer grammar t;\n"+ 98 "A : 'a' B 'c' ;\n" + 99 "B : 'b' ;\n"); 100 final int Atype = g.getTokenType("A"); 101 Interpreter engine = new Interpreter(g, new ANTLRStringStream("abc")); // should ignore the x 102 Token result = engine.scan("A"); 103 assertEquals(result.getType(), Atype); 104 } 105 testSimpleLoop()106 @Test public void testSimpleLoop() throws Exception { 107 Grammar g = new Grammar( 108 "lexer grammar t;\n"+ 109 "INT : (DIGIT)+ ;\n"+ 110 "fragment DIGIT : '0'..'9';\n"); 111 final int INTtype = g.getTokenType("INT"); 112 Interpreter engine = new Interpreter(g, new ANTLRStringStream("12x")); // should ignore the x 113 Token result = engine.scan("INT"); 114 assertEquals(result.getType(), INTtype); 115 engine = new Interpreter(g, new ANTLRStringStream("1234")); 116 result = engine.scan("INT"); 117 assertEquals(result.getType(), INTtype); 118 } 119 testMultAltLoop()120 @Test public void testMultAltLoop() throws Exception { 121 Grammar g = new Grammar( 122 "lexer grammar t;\n"+ 123 "A : ('0'..'9'|'a'|'b')+ ;\n"); 124 final int Atype = g.getTokenType("A"); 125 Interpreter engine = new Interpreter(g, new ANTLRStringStream("a")); 126 Token result = engine.scan("A"); 127 engine = new Interpreter(g, new ANTLRStringStream("a")); 128 result = engine.scan("A"); 129 assertEquals(result.getType(), Atype); 130 engine = new Interpreter(g, new ANTLRStringStream("1234")); 131 result = engine.scan("A"); 132 assertEquals(result.getType(), Atype); 133 engine = new Interpreter(g, new ANTLRStringStream("aaa")); 134 result = engine.scan("A"); 135 assertEquals(result.getType(), Atype); 136 engine = new Interpreter(g, new ANTLRStringStream("aaaa9")); 137 result = engine.scan("A"); 138 assertEquals(result.getType(), Atype); 139 engine = new Interpreter(g, new ANTLRStringStream("b")); 140 result = engine.scan("A"); 141 assertEquals(result.getType(), Atype); 142 engine = new Interpreter(g, new ANTLRStringStream("baa")); 143 result = engine.scan("A"); 144 assertEquals(result.getType(), Atype); 145 } 146 testSimpleLoops()147 @Test public void testSimpleLoops() throws Exception { 148 Grammar g = new Grammar( 149 "lexer grammar t;\n"+ 150 "A : ('0'..'9')+ '.' ('0'..'9')* | ('0'..'9')+ ;\n"); 151 final int Atype = g.getTokenType("A"); 152 CharStream input = new ANTLRStringStream("1234.5"); 153 Interpreter engine = new Interpreter(g, input); 154 Token result = engine.scan("A"); 155 assertEquals(Atype, result.getType()); 156 } 157 testTokensRules()158 @Test public void testTokensRules() throws Exception { 159 Grammar pg = new Grammar( 160 "parser grammar p;\n"+ 161 "a : (INT|FLOAT|WS)+;\n"); 162 Grammar g = new Grammar(); 163 g.importTokenVocabulary(pg); 164 g.setFileName("<string>"); 165 g.setGrammarContent( 166 "lexer grammar t;\n"+ 167 "INT : (DIGIT)+ ;\n"+ 168 "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n"+ 169 "fragment DIGIT : '0'..'9';\n" + 170 "WS : (' ')+ {channel=99;};\n"); 171 CharStream input = new ANTLRStringStream("123 139.52"); 172 Interpreter lexEngine = new Interpreter(g, input); 173 174 CommonTokenStream tokens = new CommonTokenStream(lexEngine); 175 tokens.LT(5); // make sure it grabs all tokens 176 String result = tokens.toString(); 177 //System.out.println(result); 178 String expecting = "123 139.52"; 179 assertEquals(expecting, result); 180 } 181 182 } 183