1/** The most common stream of tokens is one where every token is buffered up 2 * and tokens are prefiltered for a certain channel (the parser will only 3 * see these tokens and cannot change the filter channel number during the 4 * parse). 5 * 6 * TODO: how to access the full token stream? How to track all tokens matched per rule? 7 */ 8org.antlr.runtime.CommonTokenStream = function(tokenSource, channel) { 9 this.p = -1; 10 this.channel = org.antlr.runtime.Token.DEFAULT_CHANNEL; 11 this.v_discardOffChannelTokens = false; 12 13 this.tokens = []; 14 if (arguments.length >= 2) { 15 this.channel = channel; 16 } else if (arguments.length === 1) { 17 this.tokenSource = tokenSource; 18 } 19}; 20org.antlr.runtime.TokenStream = function() {}; 21 22org.antlr.lang.extend(org.antlr.runtime.CommonTokenStream, 23 org.antlr.runtime.TokenStream, 24{ 25 /** Reset this token stream by setting its token source. */ 26 setTokenSource: function(tokenSource) { 27 this.tokenSource = tokenSource; 28 this.tokens = []; 29 this.p = -1; 30 this.channel = org.antlr.runtime.Token.DEFAULT_CHANNEL; 31 }, 32 33 /** Load all tokens from the token source and put in tokens. 34 * This is done upon first LT request because you might want to 35 * set some token type / channel overrides before filling buffer. 36 */ 37 fillBuffer: function() { 38 var index = 0, 39 t = this.tokenSource.nextToken(), 40 discard, 41 channelI; 42 while ( org.antlr.lang.isValue(t) && 43 t.getType()!=org.antlr.runtime.CharStream.EOF ) 44 { 45 discard = false; 46 // is there a channel override for token type? 47 if ( this.channelOverrideMap ) { 48 channelI = this.channelOverrideMap[t.getType()]; 49 if ( org.antlr.lang.isValue(channelI) ) { 50 t.setChannel(channelI); 51 } 52 } 53 if ( this.discardSet && this.discardSet[t.getType()] ) 54 { 55 discard = true; 56 } 57 else if ( this.v_discardOffChannelTokens && 58 t.getChannel()!=this.channel ) 59 { 60 discard = true; 61 } 62 if ( !discard ) { 63 t.setTokenIndex(index); 64 this.tokens.push(t); 65 index++; 66 } 67 t = this.tokenSource.nextToken(); 68 } 69 // leave p pointing at first token on channel 70 this.p = 0; 71 this.p = this.skipOffTokenChannels(this.p); 72 }, 73 74 /** Move the input pointer to the next incoming token. The stream 75 * must become active with LT(1) available. consume() simply 76 * moves the input pointer so that LT(1) points at the next 77 * input symbol. Consume at least one token. 78 * 79 * Walk past any token not on the channel the parser is listening to. 80 */ 81 consume: function() { 82 if ( this.p<this.tokens.length ) { 83 this.p++; 84 this.p = this.skipOffTokenChannels(this.p); // leave p on valid token 85 } 86 }, 87 88 /** Given a starting index, return the index of the first on-channel 89 * token. 90 */ 91 skipOffTokenChannels: function(i) { 92 var n = this.tokens.length; 93 while ( i<n && (this.tokens[i]).getChannel()!=this.channel ) { 94 i++; 95 } 96 return i; 97 }, 98 99 skipOffTokenChannelsReverse: function(i) { 100 while ( i>=0 && (this.tokens[i]).getChannel()!=this.channel ) { 101 i--; 102 } 103 return i; 104 }, 105 106 /** A simple filter mechanism whereby you can tell this token stream 107 * to force all tokens of type ttype to be on channel. For example, 108 * when interpreting, we cannot exec actions so we need to tell 109 * the stream to force all WS and NEWLINE to be a different, ignored 110 * channel. 111 */ 112 setTokenTypeChannel: function(ttype, channel) { 113 if ( !this.channelOverrideMap ) { 114 this.channelOverrideMap = {}; 115 } 116 this.channelOverrideMap[ttype] = channel; 117 }, 118 119 discardTokenType: function(ttype) { 120 if ( !this.discardSet ) { 121 this.discardSet = {}; 122 } 123 this.discardSet[ttype] = true; 124 }, 125 126 discardOffChannelTokens: function(b) { 127 this.v_discardOffChannelTokens = b; 128 }, 129 130 /** Given a start and stop index, return a List of all tokens in 131 * the token type BitSet. Return null if no tokens were found. This 132 * method looks at both on and off channel tokens. 133 */ 134 getTokens: function(start, stop, types) { 135 if ( this.p === -1 ) { 136 this.fillBuffer(); 137 } 138 139 if (arguments.length===0) { 140 return this.tokens; 141 } 142 143 if (org.antlr.lang.isArray(types)) { 144 types = new org.antlr.runtime.BitSet(types); 145 } else if (org.antlr.lang.isNumber(types)) { 146 types = org.antlr.runtime.BitSet.of(types); 147 } 148 149 if ( stop>=this.tokens.length ) { 150 stop=this.tokens.length-1; 151 } 152 if ( start<0 ) { 153 start=0; 154 } 155 if ( start>stop ) { 156 return null; 157 } 158 159 // list = tokens[start:stop]:{Token t, t.getType() in types} 160 var filteredTokens = [], 161 i, 162 t; 163 for (i=start; i<=stop; i++) { 164 t = this.tokens[i]; 165 if ( !this.types || types.member(t.getType()) ) { 166 filteredTokens.push(t); 167 } 168 } 169 if ( filteredTokens.length===0 ) { 170 filteredTokens = null; 171 } 172 return filteredTokens; 173 }, 174 175 /** Get the ith token from the current position 1..n where k=1 is the 176 * first symbol of lookahead. 177 */ 178 LT: function(k) { 179 if ( this.p === -1 ) { 180 this.fillBuffer(); 181 } 182 if ( k===0 ) { 183 return null; 184 } 185 if ( k<0 ) { 186 return this.LB(-1*k); 187 } 188 if ( (this.p+k-1) >= this.tokens.length ) { 189 return org.antlr.runtime.Token.EOF_TOKEN; 190 } 191 var i = this.p, 192 n = 1; 193 // find k good tokens 194 while ( n<k ) { 195 // skip off-channel tokens 196 i = this.skipOffTokenChannels(i+1); // leave p on valid token 197 n++; 198 } 199 if ( i>=this.tokens.length ) { 200 return org.antlr.runtime.Token.EOF_TOKEN; 201 } 202 return this.tokens[i]; 203 }, 204 205 /** Look backwards k tokens on-channel tokens */ 206 LB: function(k) { 207 if ( this.p === -1 ) { 208 this.fillBuffer(); 209 } 210 if ( k===0 ) { 211 return null; 212 } 213 if ( (this.p-k)<0 ) { 214 return null; 215 } 216 217 var i = this.p, 218 n = 1; 219 // find k good tokens looking backwards 220 while ( n<=k ) { 221 // skip off-channel tokens 222 i = this.skipOffTokenChannelsReverse(i-1); // leave p on valid token 223 n++; 224 } 225 if ( i<0 ) { 226 return null; 227 } 228 return this.tokens[i]; 229 }, 230 231 /** Return absolute token i; ignore which channel the tokens are on; 232 * that is, count all tokens not just on-channel tokens. 233 */ 234 get: function(i) { 235 return this.tokens[i]; 236 }, 237 238 LA: function(i) { 239 return this.LT(i).getType(); 240 }, 241 242 mark: function() { 243 if ( this.p === -1 ) { 244 this.fillBuffer(); 245 } 246 this.lastMarker = this.index(); 247 return this.lastMarker; 248 }, 249 250 release: function(marker) { 251 // no resources to release 252 }, 253 254 size: function() { 255 return this.tokens.length; 256 }, 257 258 index: function() { 259 return this.p; 260 }, 261 262 rewind: function(marker) { 263 if (!org.antlr.lang.isNumber(marker)) { 264 marker = this.lastMarker; 265 } 266 this.seek(marker); 267 }, 268 269 reset: function() { 270 this.p = 0; 271 this.lastMarker = 0; 272 }, 273 274 seek: function(index) { 275 this.p = index; 276 }, 277 278 getTokenSource: function() { 279 return this.tokenSource; 280 }, 281 282 getSourceName: function() { 283 return this.getTokenSource().getSourceName(); 284 }, 285 286 toString: function(start, stop) { 287 if (arguments.length===0) { 288 if ( this.p === -1 ) { 289 this.fillBuffer(); 290 } 291 start = 0; 292 stop = this.tokens.length-1; 293 } 294 295 if (!org.antlr.lang.isNumber(start) && !org.antlr.lang.isNumber(stop)) { 296 if ( org.antlr.lang.isValue(start) && org.antlr.lang.isValue(stop) ) { 297 start = start.getTokenIndex(); 298 stop = stop.getTokenIndex(); 299 } else { 300 return null; 301 } 302 } 303 304 var buf = "", 305 i, 306 t; 307 308 if ( start<0 || stop<0 ) { 309 return null; 310 } 311 if ( this.p == -1 ) { 312 this.fillBuffer(); 313 } 314 if ( stop>=this.tokens.length ) { 315 stop = this.tokens.length-1; 316 } 317 for (i = start; i <= stop; i++) { 318 t = this.tokens[i]; 319 buf = buf + this.tokens[i].getText(); 320 } 321 return buf; 322 } 323}); 324