1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 package org.antlr.test; 30 31 import org.antlr.runtime.*; 32 import org.antlr.tool.Grammar; 33 import org.antlr.tool.Interpreter; 34 import org.junit.Test; 35 36 import static org.junit.Assert.*; 37 38 /** This actually tests new (12/4/09) buffered but on-demand fetching stream */ 39 public class TestCommonTokenStream extends BaseTest { testFirstToken()40 @Test public void testFirstToken() throws Exception { 41 Grammar g = new Grammar( 42 "lexer grammar t;\n"+ 43 "ID : 'a'..'z'+;\n" + 44 "INT : '0'..'9'+;\n" + 45 "SEMI : ';';\n" + 46 "ASSIGN : '=';\n" + 47 "PLUS : '+';\n" + 48 "MULT : '*';\n" + 49 "WS : ' '+;\n"); 50 // Tokens: 012345678901234567 51 // Input: x = 3 * 0 + 2 * 0; 52 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 53 Interpreter lexEngine = new Interpreter(g, input); 54 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 55 56 String result = tokens.LT(1).getText(); 57 String expecting = "x"; 58 assertEquals(expecting, result); 59 } 60 test2ndToken()61 @Test public void test2ndToken() throws Exception { 62 Grammar g = new Grammar( 63 "lexer grammar t;\n"+ 64 "ID : 'a'..'z'+;\n" + 65 "INT : '0'..'9'+;\n" + 66 "SEMI : ';';\n" + 67 "ASSIGN : '=';\n" + 68 "PLUS : '+';\n" + 69 "MULT : '*';\n" + 70 "WS : ' '+;\n"); 71 // Tokens: 012345678901234567 72 // Input: x = 3 * 0 + 2 * 0; 73 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 74 Interpreter lexEngine = new Interpreter(g, input); 75 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 76 77 String result = tokens.LT(2).getText(); 78 String expecting = " "; 79 assertEquals(expecting, result); 80 } 81 testCompleteBuffer()82 @Test public void testCompleteBuffer() throws Exception { 83 Grammar g = new Grammar( 84 "lexer grammar t;\n"+ 85 "ID : 'a'..'z'+;\n" + 86 "INT : '0'..'9'+;\n" + 87 "SEMI : ';';\n" + 88 "ASSIGN : '=';\n" + 89 "PLUS : '+';\n" + 90 "MULT : '*';\n" + 91 "WS : ' '+;\n"); 92 // Tokens: 012345678901234567 93 // Input: x = 3 * 0 + 2 * 0; 94 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 95 Interpreter lexEngine = new Interpreter(g, input); 96 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 97 98 int i = 1; 99 Token t = tokens.LT(i); 100 while ( t.getType()!=Token.EOF ) { 101 i++; 102 t = tokens.LT(i); 103 } 104 tokens.LT(i++); // push it past end 105 tokens.LT(i++); 106 107 String result = tokens.toString(); 108 String expecting = "x = 3 * 0 + 2 * 0;"; 109 assertEquals(expecting, result); 110 } 111 testCompleteBufferAfterConsuming()112 @Test public void testCompleteBufferAfterConsuming() throws Exception { 113 Grammar g = new Grammar( 114 "lexer grammar t;\n"+ 115 "ID : 'a'..'z'+;\n" + 116 "INT : '0'..'9'+;\n" + 117 "SEMI : ';';\n" + 118 "ASSIGN : '=';\n" + 119 "PLUS : '+';\n" + 120 "MULT : '*';\n" + 121 "WS : ' '+;\n"); 122 // Tokens: 012345678901234567 123 // Input: x = 3 * 0 + 2 * 0; 124 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 125 Interpreter lexEngine = new Interpreter(g, input); 126 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 127 128 Token t = tokens.LT(1); 129 while ( t.getType()!=Token.EOF ) { 130 tokens.consume(); 131 t = tokens.LT(1); 132 } 133 tokens.consume(); 134 tokens.LT(1); // push it past end 135 tokens.consume(); 136 tokens.LT(1); 137 138 String result = tokens.toString(); 139 String expecting = "x = 3 * 0 + 2 * 0;"; 140 assertEquals(expecting, result); 141 } 142 testLookback()143 @Test public void testLookback() throws Exception { 144 Grammar g = new Grammar( 145 "lexer grammar t;\n"+ 146 "ID : 'a'..'z'+;\n" + 147 "INT : '0'..'9'+;\n" + 148 "SEMI : ';';\n" + 149 "ASSIGN : '=';\n" + 150 "PLUS : '+';\n" + 151 "MULT : '*';\n" + 152 "WS : ' '+;\n"); 153 // Tokens: 012345678901234567 154 // Input: x = 3 * 0 + 2 * 0; 155 CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); 156 Interpreter lexEngine = new Interpreter(g, input); 157 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); 158 159 tokens.consume(); // get x into buffer 160 Token t = tokens.LT(-1); 161 assertEquals("x", t.getText()); 162 163 tokens.consume(); 164 tokens.consume(); // consume '=' 165 t = tokens.LT(-3); 166 assertEquals("x", t.getText()); 167 t = tokens.LT(-2); 168 assertEquals(" ", t.getText()); 169 t = tokens.LT(-1); 170 assertEquals("=", t.getText()); 171 } 172 testOffChannel()173 @Test public void testOffChannel() throws Exception { 174 TokenSource lexer = // simulate input " x =34 ;\n" 175 new TokenSource() { 176 int i = 0; 177 Token[] tokens = { 178 new CommonToken(1," "), 179 new CommonToken(1,"x"), 180 new CommonToken(1," "), 181 new CommonToken(1,"="), 182 new CommonToken(1,"34"), 183 new CommonToken(1," "), 184 new CommonToken(1," "), 185 new CommonToken(1,";"), 186 new CommonToken(1,"\n"), 187 new CommonToken(Token.EOF,"") 188 }; 189 { 190 tokens[0].setChannel(Lexer.HIDDEN); 191 tokens[2].setChannel(Lexer.HIDDEN); 192 tokens[5].setChannel(Lexer.HIDDEN); 193 tokens[6].setChannel(Lexer.HIDDEN); 194 tokens[8].setChannel(Lexer.HIDDEN); 195 } 196 @Override 197 public Token nextToken() { 198 return tokens[i++]; 199 } 200 @Override 201 public String getSourceName() { return "test"; } 202 }; 203 204 CommonTokenStream tokens = new CommonTokenStream(lexer); 205 206 assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token 207 tokens.consume(); 208 assertEquals("=", tokens.LT(1).getText()); 209 assertEquals("x", tokens.LT(-1).getText()); 210 211 tokens.consume(); 212 assertEquals("34", tokens.LT(1).getText()); 213 assertEquals("=", tokens.LT(-1).getText()); 214 215 tokens.consume(); 216 assertEquals(";", tokens.LT(1).getText()); 217 assertEquals("34", tokens.LT(-1).getText()); 218 219 tokens.consume(); 220 assertEquals(Token.EOF, tokens.LA(1)); 221 assertEquals(";", tokens.LT(-1).getText()); 222 223 assertEquals("34", tokens.LT(-2).getText()); 224 assertEquals("=", tokens.LT(-3).getText()); 225 assertEquals("x", tokens.LT(-4).getText()); 226 } 227 }