• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /// \file
2 /// Default implementation of CommonTokenStream
3 ///
4 
5 // [The "BSD licence"]
6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
7 // http://www.temporal-wave.com
8 // http://www.linkedin.com/in/jimidle
9 //
10 // All rights reserved.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions
14 // are met:
15 // 1. Redistributions of source code must retain the above copyright
16 //    notice, this list of conditions and the following disclaimer.
17 // 2. Redistributions in binary form must reproduce the above copyright
18 //    notice, this list of conditions and the following disclaimer in the
19 //    documentation and/or other materials provided with the distribution.
20 // 3. The name of the author may not be used to endorse or promote products
21 //    derived from this software without specific prior written permission.
22 //
23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 
34 #include    <antlr3tokenstream.h>
35 
36 #ifdef	ANTLR3_WINDOWS
37 #pragma warning( disable : 4100 )
38 #endif
39 
40 // COMMON_TOKEN_STREAM API
41 //
42 static void		setTokenTypeChannel     (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
43 static void		discardTokenType        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
44 static void		discardOffChannel       (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
45 static pANTLR3_VECTOR	getTokens	        (pANTLR3_COMMON_TOKEN_STREAM cts);
46 static pANTLR3_LIST	getTokenRange	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
47 static pANTLR3_LIST	getTokensSet	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
48 static pANTLR3_LIST	getTokensList	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
49 static pANTLR3_LIST	getTokensType	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
50 static void             reset                   (pANTLR3_COMMON_TOKEN_STREAM cts);
51 
52 // TOKEN_STREAM API
53 //
54 static pANTLR3_COMMON_TOKEN tokLT	        (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
55 static pANTLR3_COMMON_TOKEN dbgTokLT		(pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
56 static pANTLR3_COMMON_TOKEN get			(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
57 static pANTLR3_TOKEN_SOURCE getTokenSource	(pANTLR3_TOKEN_STREAM ts);
58 static void		    setTokenSource	(pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
59 static pANTLR3_STRING	    toString		(pANTLR3_TOKEN_STREAM ts);
60 static pANTLR3_STRING	    toStringSS		(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
61 static pANTLR3_STRING	    toStringTT		(pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
62 static void		    setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
63 
64 // INT STREAM API
65 //
66 static void		    consume			(pANTLR3_INT_STREAM is);
67 static void		    dbgConsume			(pANTLR3_INT_STREAM is);
68 static ANTLR3_UINT32	    _LA				(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
69 static ANTLR3_UINT32	    dbgLA			(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
70 static ANTLR3_MARKER	    mark			(pANTLR3_INT_STREAM is);
71 static ANTLR3_MARKER	    dbgMark			(pANTLR3_INT_STREAM is);
72 static void		    release			(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
73 static ANTLR3_UINT32	    size			(pANTLR3_INT_STREAM is);
74 static ANTLR3_MARKER	    tindex			(pANTLR3_INT_STREAM is);
75 static void		    rewindStream		(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
76 static void		    dbgRewindStream		(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
77 static void		    rewindLast			(pANTLR3_INT_STREAM is);
78 static void		    dbgRewindLast		(pANTLR3_INT_STREAM is);
79 static void		    seek			(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
80 static void		    dbgSeek			(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
81 static pANTLR3_STRING	    getSourceName		(pANTLR3_INT_STREAM is);
82 static void		    antlr3TokenStreamFree	(pANTLR3_TOKEN_STREAM	    stream);
83 static void		    antlr3CTSFree		(pANTLR3_COMMON_TOKEN_STREAM    stream);
84 
85 // Helpers
86 //
87 static void		    fillBuffer			(pANTLR3_COMMON_TOKEN_STREAM tokenStream);
88 static ANTLR3_UINT32	    skipOffTokenChannels	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
89 static ANTLR3_UINT32	    skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
90 static pANTLR3_COMMON_TOKEN LB				(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
91 
92 ANTLR3_API pANTLR3_TOKEN_STREAM
antlr3TokenStreamNew()93 antlr3TokenStreamNew()
94 {
95     pANTLR3_TOKEN_STREAM stream;
96 
97     // Memory for the interface structure
98     //
99     stream  = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
100 
101     if	(stream == NULL)
102     {
103 		return	NULL;
104     }
105 
106     // Install basic API
107     //
108     stream->free    =  antlr3TokenStreamFree;
109 
110 
111     return stream;
112 }
113 
114 static void
antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)115 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
116 {
117     ANTLR3_FREE(stream);
118 }
119 
120 static void
antlr3CTSFree(pANTLR3_COMMON_TOKEN_STREAM stream)121 antlr3CTSFree	    (pANTLR3_COMMON_TOKEN_STREAM stream)
122 {
123 	// We only free up our subordinate interfaces if they belong
124 	// to us, otherwise we let whoever owns them deal with them.
125 	//
126 	if	(stream->tstream->super == stream)
127 	{
128 		if	(stream->tstream->istream->super == stream->tstream)
129 		{
130 			stream->tstream->istream->free(stream->tstream->istream);
131 			stream->tstream->istream = NULL;
132 		}
133 		stream->tstream->free(stream->tstream);
134 	}
135 
136 	// Now we free our own resources
137 	//
138 	if	(stream->tokens != NULL)
139 	{
140 		stream->tokens->free(stream->tokens);
141 		stream->tokens	= NULL;
142 	}
143 	if	(stream->discardSet != NULL)
144 	{
145 		stream->discardSet->free(stream->discardSet);
146 		stream->discardSet  = NULL;
147 	}
148 	if	(stream->channelOverrides != NULL)
149 	{
150 		stream->channelOverrides->free(stream->channelOverrides);
151 		stream->channelOverrides = NULL;
152 	}
153 
154 	// Free our memory now
155 	//
156 	ANTLR3_FREE(stream);
157 }
158 
159 // Reset a token stream so it can be used again and can reuse it's
160 // resources.
161 //
162 static void
reset(pANTLR3_COMMON_TOKEN_STREAM cts)163 reset   (pANTLR3_COMMON_TOKEN_STREAM cts)
164 {
165 
166     // Free any resources that ar most like specifc to the
167     // run we just did.
168     //
169     if	(cts->discardSet != NULL)
170     {
171         cts->discardSet->free(cts->discardSet);
172         cts->discardSet  = NULL;
173     }
174     if	(cts->channelOverrides != NULL)
175     {
176         cts->channelOverrides->free(cts->channelOverrides);
177         cts->channelOverrides = NULL;
178     }
179 
180     // Now, if there were any existing tokens in the stream,
181     // then we just reset the vector count so that it starts
182     // again. We must traverse the entries unfortunately as
183     // there may be free pointers for custom token types and
184     // so on. However that is just a quick NULL check on the
185     // vector entries.
186     //
187     if	(cts->tokens != NULL)
188     {
189         cts->tokens->clear(cts->tokens);
190     }
191     else
192     {
193         /* Install the token tracking tables
194          */
195         cts->tokens  = antlr3VectorNew(0);
196     }
197 
198     // Reset to defaults
199     //
200     cts->discardOffChannel  = ANTLR3_FALSE;
201     cts->channel            = ANTLR3_TOKEN_DEFAULT_CHANNEL;
202     cts->p	            = -1;
203 }
204 
205 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint,pANTLR3_TOKEN_SOURCE source,pANTLR3_DEBUG_EVENT_LISTENER debugger)206 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
207 {
208     pANTLR3_COMMON_TOKEN_STREAM	stream;
209 
210 	// Create a standard token stream
211 	//
212 	stream = antlr3CommonTokenStreamSourceNew(hint, source);
213 
214 	// Install the debugger object
215 	//
216 	stream->tstream->debugger = debugger;
217 
218 	// Override standard token stream methods with debugging versions
219 	//
220 	stream->tstream->initialStreamState	= ANTLR3_FALSE;
221 
222 	stream->tstream->_LT				= dbgTokLT;
223 
224 	stream->tstream->istream->consume		= dbgConsume;
225 	stream->tstream->istream->_LA			= dbgLA;
226 	stream->tstream->istream->mark			= dbgMark;
227 	stream->tstream->istream->rewind		= dbgRewindStream;
228 	stream->tstream->istream->rewindLast	= dbgRewindLast;
229 	stream->tstream->istream->seek			= dbgSeek;
230 
231 	return stream;
232 }
233 
234 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint,pANTLR3_TOKEN_SOURCE source)235 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
236 {
237     pANTLR3_COMMON_TOKEN_STREAM	stream;
238 
239     stream = antlr3CommonTokenStreamNew(hint);
240 
241     stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
242 
243     stream->channelOverrides	= NULL;
244     stream->discardSet		= NULL;
245     stream->discardOffChannel	= ANTLR3_FALSE;
246 
247     stream->tstream->setTokenSource(stream->tstream, source);
248 
249     stream->free		=  antlr3CTSFree;
250     return  stream;
251 }
252 
253 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)254 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
255 {
256     pANTLR3_COMMON_TOKEN_STREAM stream;
257 
258     /* Memory for the interface structure
259      */
260     stream  = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
261 
262     if	(stream == NULL)
263     {
264 	return	NULL;
265     }
266 
267     /* Create space for the token stream interface
268      */
269     stream->tstream	    = antlr3TokenStreamNew();
270     stream->tstream->super  =  stream;
271 
272     /* Create space for the INT_STREAM interfacce
273      */
274     stream->tstream->istream		    =  antlr3IntStreamNew();
275     stream->tstream->istream->super	    =  (stream->tstream);
276     stream->tstream->istream->type	    = ANTLR3_TOKENSTREAM;
277 
278     /* Install the token tracking tables
279      */
280     stream->tokens  = antlr3VectorNew(0);
281 
282     /* Defaults
283      */
284     stream->p	    = -1;
285 
286     /* Install the common token stream API
287      */
288     stream->setTokenTypeChannel	    = setTokenTypeChannel;
289     stream->discardTokenType	    = discardTokenType;
290     stream->discardOffChannelToks   = discardOffChannel;
291     stream->getTokens		    = getTokens;
292     stream->getTokenRange	    = getTokenRange;
293     stream->getTokensSet	    = getTokensSet;
294     stream->getTokensList	    = getTokensList;
295     stream->getTokensType	    = getTokensType;
296     stream->reset                   = reset;
297 
298     /* Install the token stream API
299      */
300     stream->tstream->_LT			=  tokLT;
301     stream->tstream->get			=  get;
302     stream->tstream->getTokenSource	        =  getTokenSource;
303     stream->tstream->setTokenSource	        =  setTokenSource;
304     stream->tstream->toString		        =  toString;
305     stream->tstream->toStringSS		        =  toStringSS;
306     stream->tstream->toStringTT		        =  toStringTT;
307     stream->tstream->setDebugListener           =  setDebugListener;
308 
309     /* Install INT_STREAM interface
310      */
311     stream->tstream->istream->_LA	=  _LA;
312     stream->tstream->istream->mark	=  mark;
313     stream->tstream->istream->release	=  release;
314     stream->tstream->istream->size	=  size;
315     stream->tstream->istream->index	=  tindex;
316     stream->tstream->istream->rewind	=  rewindStream;
317     stream->tstream->istream->rewindLast=  rewindLast;
318     stream->tstream->istream->seek	=  seek;
319     stream->tstream->istream->consume	=  consume;
320     stream->tstream->istream->getSourceName = getSourceName;
321 
322     return  stream;
323 }
324 
325 // Install a debug listener adn switch to debug mode methods
326 //
327 static void
setDebugListener(pANTLR3_TOKEN_STREAM ts,pANTLR3_DEBUG_EVENT_LISTENER debugger)328 setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
329 {
330 		// Install the debugger object
331 	//
332 	ts->debugger = debugger;
333 
334 	// Override standard token stream methods with debugging versions
335 	//
336 	ts->initialStreamState	= ANTLR3_FALSE;
337 
338 	ts->_LT				= dbgTokLT;
339 
340 	ts->istream->consume		= dbgConsume;
341 	ts->istream->_LA			= dbgLA;
342 	ts->istream->mark			= dbgMark;
343 	ts->istream->rewind			= dbgRewindStream;
344 	ts->istream->rewindLast		= dbgRewindLast;
345 	ts->istream->seek			= dbgSeek;
346 }
347 
348 /** Get the ith token from the current position 1..n where k=1 is the
349 *  first symbol of lookahead.
350 */
351 static pANTLR3_COMMON_TOKEN
tokLT(pANTLR3_TOKEN_STREAM ts,ANTLR3_INT32 k)352 tokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
353 {
354 	ANTLR3_INT32    i;
355 	ANTLR3_INT32    n;
356 	pANTLR3_COMMON_TOKEN_STREAM cts;
357 
358 	cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
359 
360         if	(k < 0)
361 	{
362 		return LB(cts, -k);
363 	}
364 
365 	if	(cts->p == -1)
366 	{
367 		fillBuffer(cts);
368 	}
369 
370         // Here we used to check for k == 0 and return 0, but this seems
371         // a superfluous check to me. LT(k=0) is therefore just undefined
372         // and we won't waste the clock cycles on the check
373         //
374 
375 	if	((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
376 	{
377 		pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
378 
379 		teof->setStartIndex (teof, ts->istream->index	    (ts->istream));
380 		teof->setStopIndex  (teof, ts->istream->index	    (ts->istream));
381 		return  teof;
382 	}
383 
384 	i	= cts->p;
385 	n	= 1;
386 
387 	/* Need to find k good tokens, skipping ones that are off channel
388 	*/
389 	while   ( n < k)
390 	{
391 		/* Skip off-channel tokens */
392 		i = skipOffTokenChannels(cts, i+1); /* leave p on valid token    */
393 		n++;
394 	}
395 	if	( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
396 	{
397 		pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
398 
399 		teof->setStartIndex (teof, ts->istream->index(ts->istream));
400 		teof->setStopIndex  (teof, ts->istream->index(ts->istream));
401 		return  teof;
402 	}
403 
404 	// Here the token must be in the input vector. Rather then incur
405 	// function call penalty, we just return the pointer directly
406 	// from the vector
407 	//
408 	return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
409 	//return  (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
410 }
411 
412 /// Debug only method to flag consumption of initial off-channel
413 /// tokens in the input stream
414 ///
415 static void
consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)416 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
417 {
418 	ANTLR3_MARKER	first;
419 	ANTLR3_INT32	i;
420 	pANTLR3_TOKEN_STREAM	ts;
421 
422 	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
423 	first	= is->index(is);
424 
425 	for	(i=0; i<first; i++)
426 	{
427 		ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
428 	}
429 
430 	ts->initialStreamState = ANTLR3_FALSE;
431 
432 }
433 
434 /// As per the normal tokLT but sends information to the debugger
435 ///
436 static pANTLR3_COMMON_TOKEN
dbgTokLT(pANTLR3_TOKEN_STREAM ts,ANTLR3_INT32 k)437 dbgTokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
438 {
439 	if	(ts->initialStreamState == ANTLR3_TRUE)
440 	{
441 		consumeInitialHiddenTokens(ts->istream);
442 	}
443 	return tokLT(ts, k);
444 }
445 
446 #ifdef	ANTLR3_WINDOWS
447 	/* When fully optimized VC7 complains about non reachable code.
448 	 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
449 	 */
450 #pragma warning( disable : 4702 )
451 #endif
452 
453 static pANTLR3_COMMON_TOKEN
LB(pANTLR3_COMMON_TOKEN_STREAM cts,ANTLR3_INT32 k)454 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
455 {
456     ANTLR3_INT32 i;
457     ANTLR3_INT32 n;
458 
459     if (cts->p == -1)
460     {
461         fillBuffer(cts);
462     }
463     if (k == 0)
464     {
465         return NULL;
466     }
467     if ((cts->p - k) < 0)
468     {
469         return NULL;
470     }
471 
472     i = cts->p;
473     n = 1;
474 
475     /* Need to find k good tokens, going backwards, skipping ones that are off channel
476      */
477     while (n <= (ANTLR3_INT32) k)
478     {
479         /* Skip off-channel tokens
480          */
481 
482         i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token    */
483         n++;
484     }
485     if (i < 0)
486     {
487         return NULL;
488     }
489 	// Here the token must be in the input vector. Rather then incut
490 	// function call penalty, we jsut return the pointer directly
491 	// from the vector
492 	//
493 	return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
494 }
495 
496 static pANTLR3_COMMON_TOKEN
get(pANTLR3_TOKEN_STREAM ts,ANTLR3_UINT32 i)497 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
498 {
499     pANTLR3_COMMON_TOKEN_STREAM cts;
500 
501     cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
502 
503     return  (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i));  /* Token index is zero based but vectors are 1 based */
504 }
505 
506 static pANTLR3_TOKEN_SOURCE
getTokenSource(pANTLR3_TOKEN_STREAM ts)507 getTokenSource	(pANTLR3_TOKEN_STREAM ts)
508 {
509     return  ts->tokenSource;
510 }
511 
512 static void
setTokenSource(pANTLR3_TOKEN_STREAM ts,pANTLR3_TOKEN_SOURCE tokenSource)513 setTokenSource	(   pANTLR3_TOKEN_STREAM ts,
514 		    pANTLR3_TOKEN_SOURCE tokenSource)
515 {
516     ts->tokenSource	= tokenSource;
517 }
518 
519 static pANTLR3_STRING
toString(pANTLR3_TOKEN_STREAM ts)520 toString    (pANTLR3_TOKEN_STREAM ts)
521 {
522     pANTLR3_COMMON_TOKEN_STREAM cts;
523 
524     cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
525 
526     if	(cts->p == -1)
527     {
528 	fillBuffer(cts);
529     }
530 
531     return  ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
532 }
533 
534 static pANTLR3_STRING
toStringSS(pANTLR3_TOKEN_STREAM ts,ANTLR3_UINT32 start,ANTLR3_UINT32 stop)535 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
536 {
537     pANTLR3_STRING string;
538     pANTLR3_TOKEN_SOURCE tsource;
539     pANTLR3_COMMON_TOKEN tok;
540     ANTLR3_UINT32 i;
541     pANTLR3_COMMON_TOKEN_STREAM cts;
542 
543     cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
544 
545     if (cts->p == -1)
546     {
547         fillBuffer(cts);
548     }
549     if (stop >= ts->istream->size(ts->istream))
550     {
551         stop = ts->istream->size(ts->istream) - 1;
552     }
553 
554     /* Who is giving us these tokens?
555      */
556     tsource = ts->getTokenSource(ts);
557 
558     if (tsource != NULL && cts->tokens != NULL)
559     {
560         /* Finally, let's get a string
561          */
562         string = tsource->strFactory->newRaw(tsource->strFactory);
563 
564         for (i = start; i <= stop; i++)
565         {
566             tok = ts->get(ts, i);
567             if (tok != NULL)
568             {
569                 string->appendS(string, tok->getText(tok));
570             }
571         }
572 
573         return string;
574     }
575     return NULL;
576 
577 }
578 
579 static pANTLR3_STRING
toStringTT(pANTLR3_TOKEN_STREAM ts,pANTLR3_COMMON_TOKEN start,pANTLR3_COMMON_TOKEN stop)580 toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
581 {
582 	if	(start != NULL && stop != NULL)
583 	{
584 		return	ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
585 	}
586 	else
587 	{
588 		return	NULL;
589 	}
590 }
591 
592 /** Move the input pointer to the next incoming token.  The stream
593  *  must become active with LT(1) available.  consume() simply
594  *  moves the input pointer so that LT(1) points at the next
595  *  input symbol. Consume at least one token.
596  *
597  *  Walk past any token not on the channel the parser is listening to.
598  */
599 static void
consume(pANTLR3_INT_STREAM is)600 consume	(pANTLR3_INT_STREAM is)
601 {
602 	pANTLR3_COMMON_TOKEN_STREAM cts;
603 	pANTLR3_TOKEN_STREAM	ts;
604 
605 	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
606 	cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
607 
608         if	((ANTLR3_UINT32)cts->p < cts->tokens->count)
609 	{
610 		cts->p++;
611 		cts->p	= skipOffTokenChannels(cts, cts->p);
612 	}
613 }
614 
615 
616 /// As per ordinary consume but notifies the debugger about hidden
617 /// tokens and so on.
618 ///
619 static void
dbgConsume(pANTLR3_INT_STREAM is)620 dbgConsume	(pANTLR3_INT_STREAM is)
621 {
622 	pANTLR3_TOKEN_STREAM	ts;
623 	ANTLR3_MARKER			a;
624 	ANTLR3_MARKER			b;
625 	pANTLR3_COMMON_TOKEN	t;
626 
627 	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
628 
629 	if	(ts->initialStreamState == ANTLR3_TRUE)
630 	{
631 		consumeInitialHiddenTokens(is);
632 	}
633 
634 	a = is->index(is);		// Where are we right now?
635 	t = ts->_LT(ts, 1);		// Current token from stream
636 
637 	consume(is);			// Standard consumer
638 
639 	b = is->index(is);		// Where are we after consuming 1 on channel token?
640 
641 	ts->debugger->consumeToken(ts->debugger, t);	// Tell the debugger that we consumed the first token
642 
643 	if	(b>a+1)
644 	{
645 		// The standard consume caused the index to advance by more than 1,
646 		// which can only happen if it skipped some off-channel tokens.
647 		// we need to tell the debugger about those tokens.
648 		//
649 		ANTLR3_MARKER	i;
650 
651 		for	(i = a+1; i<b; i++)
652 		{
653 			ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
654 		}
655 
656 	}
657 }
658 
659 /** A simple filter mechanism whereby you can tell this token stream
660  *  to force all tokens of type ttype to be on channel.  For example,
661  *  when interpreting, we cannot execute actions so we need to tell
662  *  the stream to force all WS and NEWLINE to be a different, ignored,
663  *  channel.
664  */
665 static void
setTokenTypeChannel(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 ttype,ANTLR3_UINT32 channel)666 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
667 {
668     if	(tokenStream->channelOverrides == NULL)
669     {
670 	tokenStream->channelOverrides	= antlr3ListNew(10);
671     }
672 
673     /* We add one to the channel so we can distinguish NULL as being no entry in the
674      * table for a particular token type.
675      */
676     tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
677 }
678 
679 static void
discardTokenType(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_INT32 ttype)680 discardTokenType    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
681 {
682     if	(tokenStream->discardSet == NULL)
683     {
684 	tokenStream->discardSet	= antlr3ListNew(31);
685     }
686 
687     /* We add one to the channel so we can distinguish NULL as being no entry in the
688      * table for a particular token type. We could use bitsets for this I suppose too.
689      */
690     tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
691 }
692 
693 static void
discardOffChannel(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_BOOLEAN discard)694 discardOffChannel   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
695 {
696     tokenStream->discardOffChannel  = discard;
697 }
698 
699 static pANTLR3_VECTOR
getTokens(pANTLR3_COMMON_TOKEN_STREAM tokenStream)700 getTokens   (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
701 {
702     if	(tokenStream->p == -1)
703     {
704 	fillBuffer(tokenStream);
705     }
706 
707     return  tokenStream->tokens;
708 }
709 
710 static pANTLR3_LIST
getTokenRange(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop)711 getTokenRange	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
712 {
713     return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
714 }
715 /** Given a start and stop index, return a List of all tokens in
716  *  the token type BitSet.  Return null if no tokens were found.  This
717  *  method looks at both on and off channel tokens.
718  */
719 static pANTLR3_LIST
getTokensSet(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop,pANTLR3_BITSET types)720 getTokensSet	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
721 {
722     pANTLR3_LIST	    filteredList;
723     ANTLR3_UINT32	    i;
724     ANTLR3_UINT32	    n;
725     pANTLR3_COMMON_TOKEN    tok;
726 
727     if	(tokenStream->p == -1)
728     {
729 	fillBuffer(tokenStream);
730     }
731     if	(stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
732     {
733 	stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
734     }
735     if	(start > stop)
736     {
737 	return NULL;
738     }
739 
740     /* We have the range set, now we need to iterate through the
741      * installed tokens and create a new list with just the ones we want
742      * in it. We are just moving pointers about really.
743      */
744     filteredList    = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
745 
746     for	(i = start, n = 0; i<= stop; i++)
747     {
748 	tok = tokenStream->tstream->get(tokenStream->tstream, i);
749 
750 	if  (	   types == NULL
751 		|| types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
752 	    )
753 	{
754 	    filteredList->put(filteredList, n++, (void *)tok, NULL);
755 	}
756     }
757 
758     /* Did we get any then?
759      */
760     if	(filteredList->size(filteredList) == 0)
761     {
762 	filteredList->free(filteredList);
763 	filteredList	= NULL;
764     }
765 
766     return  filteredList;
767 }
768 
769 static pANTLR3_LIST
getTokensList(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop,pANTLR3_LIST list)770 getTokensList	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
771 {
772     pANTLR3_BITSET  bitSet;
773     pANTLR3_LIST    newlist;
774 
775     bitSet  = antlr3BitsetList(list->table);
776 
777     newlist    = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
778 
779     bitSet->free(bitSet);
780 
781     return  newlist;
782 
783 }
784 
785 static pANTLR3_LIST
getTokensType(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop,ANTLR3_UINT32 type)786 getTokensType	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
787 {
788     pANTLR3_BITSET  bitSet;
789     pANTLR3_LIST    newlist;
790 
791     bitSet  = antlr3BitsetOf(type, -1);
792     newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
793 
794     bitSet->free(bitSet);
795 
796     return  newlist;
797 }
798 
799 static ANTLR3_UINT32
_LA(pANTLR3_INT_STREAM is,ANTLR3_INT32 i)800 _LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
801 {
802 	pANTLR3_TOKEN_STREAM    ts;
803 	pANTLR3_COMMON_TOKEN    tok;
804 
805 	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
806 
807 	tok	    =  ts->_LT(ts, i);
808 
809 	if	(tok != NULL)
810 	{
811 		return	tok->getType(tok);
812 	}
813 	else
814 	{
815 		return	ANTLR3_TOKEN_INVALID;
816 	}
817 }
818 
819 /// As per _LA() but for debug mode.
820 ///
821 static ANTLR3_UINT32
dbgLA(pANTLR3_INT_STREAM is,ANTLR3_INT32 i)822 dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
823 {
824     pANTLR3_TOKEN_STREAM    ts;
825 
826     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
827 
828 	if	(ts->initialStreamState == ANTLR3_TRUE)
829 	{
830 		consumeInitialHiddenTokens(is);
831 	}
832 	ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
833 	return	_LA(is, i);
834 }
835 
836 static ANTLR3_MARKER
mark(pANTLR3_INT_STREAM is)837 mark	(pANTLR3_INT_STREAM is)
838 {
839     is->lastMarker = is->index(is);
840     return  is->lastMarker;
841 }
842 
843 /// As per mark() but with a call to tell the debugger we are doing this
844 ///
845 static ANTLR3_MARKER
dbgMark(pANTLR3_INT_STREAM is)846 dbgMark	(pANTLR3_INT_STREAM is)
847 {
848     pANTLR3_TOKEN_STREAM    ts;
849 
850     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
851 
852 	is->lastMarker = is->index(is);
853 	ts->debugger->mark(ts->debugger, is->lastMarker);
854 
855     return  is->lastMarker;
856 }
857 
858 static void
release(pANTLR3_INT_STREAM is,ANTLR3_MARKER mark)859 release	(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
860 {
861     return;
862 }
863 
864 static ANTLR3_UINT32
size(pANTLR3_INT_STREAM is)865 size	(pANTLR3_INT_STREAM is)
866 {
867     pANTLR3_COMMON_TOKEN_STREAM cts;
868     pANTLR3_TOKEN_STREAM	ts;
869 
870     if (is->cachedSize > 0)
871     {
872 	return  is->cachedSize;
873     }
874     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
875     cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
876 
877     is->cachedSize =  cts->tokens->count;
878     return  is->cachedSize;
879 }
880 
881 static ANTLR3_MARKER
tindex(pANTLR3_INT_STREAM is)882 tindex	(pANTLR3_INT_STREAM is)
883 {
884     pANTLR3_COMMON_TOKEN_STREAM cts;
885     pANTLR3_TOKEN_STREAM	ts;
886 
887     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
888     cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
889 
890     return  cts->p;
891 }
892 
893 static void
dbgRewindLast(pANTLR3_INT_STREAM is)894 dbgRewindLast	(pANTLR3_INT_STREAM is)
895 {
896 	pANTLR3_TOKEN_STREAM	ts;
897 
898     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
899 
900 	ts->debugger->rewindLast(ts->debugger);
901 
902     is->rewind(is, is->lastMarker);
903 }
904 static void
rewindLast(pANTLR3_INT_STREAM is)905 rewindLast	(pANTLR3_INT_STREAM is)
906 {
907     is->rewind(is, is->lastMarker);
908 }
909 static void
rewindStream(pANTLR3_INT_STREAM is,ANTLR3_MARKER marker)910 rewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
911 {
912     is->seek(is, (ANTLR3_UINT32)(marker));
913 }
914 static void
dbgRewindStream(pANTLR3_INT_STREAM is,ANTLR3_MARKER marker)915 dbgRewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
916 {
917     pANTLR3_TOKEN_STREAM	ts;
918 
919     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
920 
921 	ts->debugger->rewind(ts->debugger, marker);
922 
923     is->seek(is, (ANTLR3_UINT32)(marker));
924 }
925 
926 static void
seek(pANTLR3_INT_STREAM is,ANTLR3_MARKER index)927 seek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
928 {
929     pANTLR3_COMMON_TOKEN_STREAM cts;
930     pANTLR3_TOKEN_STREAM	ts;
931 
932     ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
933     cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
934 
935     cts->p  = (ANTLR3_UINT32)index;
936 }
937 static void
dbgSeek(pANTLR3_INT_STREAM is,ANTLR3_MARKER index)938 dbgSeek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
939 {
940 	// TODO: Implement seek in debugger when Ter adds it to Java
941 	//
942 	seek(is, index);
943 }
944 ANTLR3_API void
fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)945 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
946 {
947     fillBuffer(tokenStream);
948 }
949 static void
fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream)950 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
951     ANTLR3_UINT32 index;
952     pANTLR3_COMMON_TOKEN tok;
953     ANTLR3_BOOLEAN discard;
954     void * channelI;
955 
956     /* Start at index 0 of course
957      */
958     index = 0;
959 
960     /* Pick out the next token from the token source
961      * Remember we just get a pointer (reference if you like) here
962      * and so if we store it anywhere, we don't set any pointers to auto free it.
963      */
964     tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
965 
966     while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
967     {
968         discard = ANTLR3_FALSE; /* Assume we are not discarding	*/
969 
970         /* I employ a bit of a trick, or perhaps hack here. Rather than
971          * store a pointer to a structure in the override map and discard set
972          * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
973          * we can distinguish "not being there" from "being channel or type 0"
974          */
975 
976         if (tokenStream->discardSet != NULL
977             && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
978         {
979             discard = ANTLR3_TRUE;
980         }
981         else if (   tokenStream->discardOffChannel == ANTLR3_TRUE
982                  && tok->getChannel(tok) != tokenStream->channel
983                  )
984         {
985             discard = ANTLR3_TRUE;
986         }
987         else if (tokenStream->channelOverrides != NULL)
988         {
989             /* See if this type is in the override map
990              */
991             channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
992 
993             if (channelI != NULL)
994             {
995                 /* Override found
996                  */
997                 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
998             }
999         }
1000 
1001         /* If not discarding it, add it to the list at the current index
1002          */
1003         if (discard == ANTLR3_FALSE)
1004         {
1005             /* Add it, indicating that we will delete it and the table should not
1006              */
1007             tok->setTokenIndex(tok, index);
1008             tokenStream->p++;
1009             tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
1010             index++;
1011         }
1012 
1013         tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
1014     }
1015 
1016     /* Cache the size so we don't keep doing indirect method calls. We do this as
1017      * early as possible so that anything after this may utilize the cached value.
1018      */
1019     tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
1020 
1021     /* Set the consume pointer to the first token that is on our channel
1022      */
1023     tokenStream->p = 0;
1024     tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
1025 
1026 }
1027 
1028 /// Given a starting index, return the index of the first on-channel
1029 ///  token.
1030 ///
1031 static ANTLR3_UINT32
skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_INT32 i)1032 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
1033     ANTLR3_INT32 n;
1034     pANTLR3_COMMON_TOKEN tok;
1035 
1036     n = tokenStream->tstream->istream->cachedSize;
1037 
1038     while (i < n)
1039     {
1040         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
1041 
1042         if (tok->channel!= tokenStream->channel)
1043         {
1044             i++;
1045         }
1046         else
1047         {
1048             return i;
1049         }
1050     }
1051     return i;
1052 }
1053 
1054 static ANTLR3_UINT32
skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_INT32 x)1055 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
1056 {
1057     pANTLR3_COMMON_TOKEN tok;
1058 
1059     while (x >= 0)
1060     {
1061         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
1062 
1063         if ((tok->channel != tokenStream->channel))
1064         {
1065             x--;
1066         }
1067         else
1068         {
1069             return x;
1070         }
1071     }
1072     return x;
1073 }
1074 
1075 /// Return a string that represents the name assoicated with the input source
1076 ///
1077 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
1078 ///
1079 /// /returns
1080 /// /implements ANTLR3_INT_STREAM_struct::getSourceName()
1081 ///
1082 static pANTLR3_STRING
getSourceName(pANTLR3_INT_STREAM is)1083 getSourceName				(pANTLR3_INT_STREAM is)
1084 {
1085 	// Slightly convoluted as we must trace back to the lexer's input source
1086 	// via the token source. The streamName that is here is not initialized
1087 	// because this is a token stream, not a file or string stream, which are the
1088 	// only things that have a context for a source name.
1089 	//
1090 	return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
1091 }
1092