• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/** The most common stream of tokens is one where every token is buffered up
2 *  and tokens are prefiltered for a certain channel (the parser will only
3 *  see these tokens and cannot change the filter channel number during the
4 *  parse).
5 *
6 *  TODO: how to access the full token stream?  How to track all tokens matched per rule?
7 */
8org.antlr.runtime.CommonTokenStream = function(tokenSource, channel) {
9    this.p = -1;
10    this.channel = org.antlr.runtime.Token.DEFAULT_CHANNEL;
11    this.v_discardOffChannelTokens = false;
12
13    this.tokens = [];
14    if (arguments.length >= 2) {
15        this.channel = channel;
16    } else if (arguments.length === 1) {
17        this.tokenSource = tokenSource;
18    }
19};
20org.antlr.runtime.TokenStream = function() {};
21
22org.antlr.lang.extend(org.antlr.runtime.CommonTokenStream,
23                      org.antlr.runtime.TokenStream,
24{
25    /** Reset this token stream by setting its token source. */
26    setTokenSource: function(tokenSource) {
27        this.tokenSource = tokenSource;
28        this.tokens = [];
29        this.p = -1;
30        this.channel = org.antlr.runtime.Token.DEFAULT_CHANNEL;
31    },
32
33    /** Load all tokens from the token source and put in tokens.
34     *  This is done upon first LT request because you might want to
35     *  set some token type / channel overrides before filling buffer.
36     */
37    fillBuffer: function() {
38        var index = 0,
39            t = this.tokenSource.nextToken(),
40            discard,
41            channelI;
42        while ( org.antlr.lang.isValue(t) &&
43                t.getType()!=org.antlr.runtime.CharStream.EOF )
44        {
45            discard = false;
46            // is there a channel override for token type?
47            if ( this.channelOverrideMap ) {
48                channelI = this.channelOverrideMap[t.getType()];
49                if ( org.antlr.lang.isValue(channelI) ) {
50                    t.setChannel(channelI);
51                }
52            }
53            if ( this.discardSet && this.discardSet[t.getType()] )
54            {
55                discard = true;
56            }
57            else if ( this.v_discardOffChannelTokens &&
58                    t.getChannel()!=this.channel )
59            {
60                discard = true;
61            }
62            if ( !discard )    {
63                t.setTokenIndex(index);
64                this.tokens.push(t);
65                index++;
66            }
67            t = this.tokenSource.nextToken();
68        }
69        // leave p pointing at first token on channel
70        this.p = 0;
71        this.p = this.skipOffTokenChannels(this.p);
72    },
73
74    /** Move the input pointer to the next incoming token.  The stream
75     *  must become active with LT(1) available.  consume() simply
76     *  moves the input pointer so that LT(1) points at the next
77     *  input symbol. Consume at least one token.
78     *
79     *  Walk past any token not on the channel the parser is listening to.
80     */
81    consume: function() {
82        if ( this.p<this.tokens.length ) {
83            this.p++;
84            this.p = this.skipOffTokenChannels(this.p); // leave p on valid token
85        }
86    },
87
88    /** Given a starting index, return the index of the first on-channel
89     *  token.
90     */
91    skipOffTokenChannels: function(i) {
92        var n = this.tokens.length;
93        while ( i<n && (this.tokens[i]).getChannel()!=this.channel ) {
94            i++;
95        }
96        return i;
97    },
98
99    skipOffTokenChannelsReverse: function(i) {
100        while ( i>=0 && (this.tokens[i]).getChannel()!=this.channel ) {
101            i--;
102        }
103        return i;
104    },
105
106    /** A simple filter mechanism whereby you can tell this token stream
107     *  to force all tokens of type ttype to be on channel.  For example,
108     *  when interpreting, we cannot exec actions so we need to tell
109     *  the stream to force all WS and NEWLINE to be a different, ignored
110     *  channel.
111     */
112    setTokenTypeChannel: function(ttype, channel) {
113        if ( !this.channelOverrideMap ) {
114            this.channelOverrideMap = {};
115        }
116        this.channelOverrideMap[ttype] = channel;
117    },
118
119    discardTokenType: function(ttype) {
120        if ( !this.discardSet ) {
121            this.discardSet = {};
122        }
123        this.discardSet[ttype] = true;
124    },
125
126    discardOffChannelTokens: function(b) {
127        this.v_discardOffChannelTokens = b;
128    },
129
130    /** Given a start and stop index, return a List of all tokens in
131     *  the token type BitSet.  Return null if no tokens were found.  This
132     *  method looks at both on and off channel tokens.
133     */
134    getTokens: function(start, stop, types) {
135        if ( this.p === -1 ) {
136            this.fillBuffer();
137        }
138
139        if (arguments.length===0) {
140            return this.tokens;
141        }
142
143        if (org.antlr.lang.isArray(types)) {
144            types = new org.antlr.runtime.BitSet(types);
145        } else if (org.antlr.lang.isNumber(types)) {
146            types = org.antlr.runtime.BitSet.of(types);
147        }
148
149        if ( stop>=this.tokens.length ) {
150            stop=this.tokens.length-1;
151        }
152        if ( start<0 ) {
153            start=0;
154        }
155        if ( start>stop ) {
156            return null;
157        }
158
159        // list = tokens[start:stop]:{Token t, t.getType() in types}
160        var filteredTokens = [],
161            i,
162            t;
163        for (i=start; i<=stop; i++) {
164            t = this.tokens[i];
165            if ( !this.types || types.member(t.getType()) ) {
166                filteredTokens.push(t);
167            }
168        }
169        if ( filteredTokens.length===0 ) {
170            filteredTokens = null;
171        }
172        return filteredTokens;
173    },
174
175    /** Get the ith token from the current position 1..n where k=1 is the
176     *  first symbol of lookahead.
177     */
178    LT: function(k) {
179        if ( this.p === -1 ) {
180            this.fillBuffer();
181        }
182        if ( k===0 ) {
183            return null;
184        }
185        if ( k<0 ) {
186            return this.LB(-1*k);
187        }
188        if ( (this.p+k-1) >= this.tokens.length ) {
189            return org.antlr.runtime.Token.EOF_TOKEN;
190        }
191        var i = this.p,
192            n = 1;
193        // find k good tokens
194        while ( n<k ) {
195            // skip off-channel tokens
196            i = this.skipOffTokenChannels(i+1); // leave p on valid token
197            n++;
198        }
199        if ( i>=this.tokens.length ) {
200            return org.antlr.runtime.Token.EOF_TOKEN;
201        }
202        return this.tokens[i];
203    },
204
205    /** Look backwards k tokens on-channel tokens */
206    LB: function(k) {
207        if ( this.p === -1 ) {
208            this.fillBuffer();
209        }
210        if ( k===0 ) {
211            return null;
212        }
213        if ( (this.p-k)<0 ) {
214            return null;
215        }
216
217        var i = this.p,
218            n = 1;
219        // find k good tokens looking backwards
220        while ( n<=k ) {
221            // skip off-channel tokens
222            i = this.skipOffTokenChannelsReverse(i-1); // leave p on valid token
223            n++;
224        }
225        if ( i<0 ) {
226            return null;
227        }
228        return this.tokens[i];
229    },
230
231    /** Return absolute token i; ignore which channel the tokens are on;
232     *  that is, count all tokens not just on-channel tokens.
233     */
234    get: function(i) {
235        return this.tokens[i];
236    },
237
238    LA: function(i) {
239        return this.LT(i).getType();
240    },
241
242    mark: function() {
243        if ( this.p === -1 ) {
244            this.fillBuffer();
245        }
246        this.lastMarker = this.index();
247        return this.lastMarker;
248    },
249
250    release: function(marker) {
251        // no resources to release
252    },
253
254    size: function() {
255        return this.tokens.length;
256    },
257
258    index: function() {
259        return this.p;
260    },
261
262    rewind: function(marker) {
263        if (!org.antlr.lang.isNumber(marker)) {
264            marker = this.lastMarker;
265        }
266        this.seek(marker);
267    },
268
269    reset: function() {
270        this.p = 0;
271        this.lastMarker = 0;
272    },
273
274    seek: function(index) {
275        this.p = index;
276    },
277
278    getTokenSource: function() {
279        return this.tokenSource;
280    },
281
282    getSourceName: function() {
283        return this.getTokenSource().getSourceName();
284    },
285
286    toString: function(start, stop) {
287        if (arguments.length===0) {
288            if ( this.p === -1 ) {
289                this.fillBuffer();
290            }
291            start = 0;
292            stop = this.tokens.length-1;
293        }
294
295        if (!org.antlr.lang.isNumber(start) && !org.antlr.lang.isNumber(stop)) {
296            if ( org.antlr.lang.isValue(start) && org.antlr.lang.isValue(stop) ) {
297                start = start.getTokenIndex();
298                stop = stop.getTokenIndex();
299            } else {
300                return null;
301            }
302        }
303
304        var buf = "",
305            i,
306            t;
307
308        if ( start<0 || stop<0 ) {
309            return null;
310        }
311        if ( this.p == -1 ) {
312            this.fillBuffer();
313        }
314        if ( stop>=this.tokens.length ) {
315            stop = this.tokens.length-1;
316        }
317        for (i = start; i <= stop; i++) {
318            t = this.tokens[i];
319            buf = buf + this.tokens[i].getText();
320        }
321        return buf;
322    }
323});
324