Sources/Antlr3.Runtime/LegacyCommonTokenStream.cs

/*
 * [The "BSD licence"]
 * Copyright (c) 2005-2008 Terence Parr
 * All rights reserved.
 *
 * Conversion to C#:
 * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

namespace Antlr.Runtime
{
    using System.Collections.Generic;

    using InvalidOperationException = System.InvalidOperationException;
    using StringBuilder = System.Text.StringBuilder;

    /** <summary>
     *  The most common stream of tokens is one where every token is buffered up
     *  and tokens are prefiltered for a certain channel (the parser will only
     *  see these tokens and cannot change the filter channel number during the
     *  parse).
     *  </summary>
     *
     *  <remarks>TODO: how to access the full token stream?  How to track all tokens matched per rule?</remarks>
     */
    [System.Serializable]
    public class LegacyCommonTokenStream : ITokenStream
    {
        [System.NonSerialized]
        ITokenSource _tokenSource;

        /** <summary>
         *  Record every single token pulled from the source so we can reproduce
         *  chunks of it later.
         *  </summary>
         */
        protected List<IToken> tokens;

        /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
        protected IDictionary<int, int> channelOverrideMap;

        /** <summary>Set<tokentype>; discard any tokens with this type</summary> */
        protected List<int> discardSet;

        /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
        protected int channel = TokenChannels.Default;

        /** <summary>By default, track all incoming tokens</summary> */
        protected bool discardOffChannelTokens = false;

        /** <summary>Track the last mark() call result value for use in rewind().</summary> */
        protected int lastMarker;

        /** <summary>
         *  The index into the tokens list of the current token (next token
         *  to consume).  p==-1 indicates that the tokens list is empty
         *  </summary>
         */
        protected int p = -1;

        public LegacyCommonTokenStream()
        {
            tokens = new List<IToken>( 500 );
        }

        public LegacyCommonTokenStream(ITokenSource tokenSource)
            : this()
        {
            this._tokenSource = tokenSource;
        }

        public LegacyCommonTokenStream( ITokenSource tokenSource, int channel )
            : this( tokenSource )
        {
            this.channel = channel;
        }

        public virtual int Index
        {
            get
            {
                return p;
            }
        }

        /// <summary>
        /// How deep have we gone?
        /// </summary>
        public virtual int Range
        {
            get;
            protected set;
        }

        /** <summary>Reset this token stream by setting its token source.</summary> */
        public virtual void SetTokenSource( ITokenSource tokenSource )
        {
            this._tokenSource = tokenSource;
            tokens.Clear();
            p = -1;
            channel = TokenChannels.Default;
        }

        /** <summary>
         *  Load all tokens from the token source and put in tokens.
         *  This is done upon first LT request because you might want to
         *  set some token type / channel overrides before filling buffer.
         *  </summary>
         */
        public virtual void FillBuffer()
        {
            // fast return if the buffer is already full
            if ( p != -1 )
                return;

            int index = 0;
            IToken t = _tokenSource.NextToken();
            while ( t != null && t.Type != CharStreamConstants.EndOfFile )
            {
                bool discard = false;
                // is there a channel override for token type?
                int channelI;
                if ( channelOverrideMap != null && channelOverrideMap.TryGetValue( t.Type, out channelI ) )
                    t.Channel = channelI;

                //if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) )
                //{
                //    object channelI = channelOverrideMap.get( t.getType() );
                //    if ( channelI != null )
                //    {
                //        t.setChannel( (int)channelI );
                //    }
                //}
                if ( discardSet != null &&
                     discardSet.Contains( t.Type ) )
                {
                    discard = true;
                }
                else if ( discardOffChannelTokens && t.Channel != this.channel )
                {
                    discard = true;
                }
                if ( !discard )
                {
                    t.TokenIndex = index;
                    tokens.Add( t );
                    index++;
                }
                t = _tokenSource.NextToken();
            }
            // leave p pointing at first token on channel
            p = 0;
            p = SkipOffTokenChannels( p );
        }

        /** <summary>
         *  Move the input pointer to the next incoming token.  The stream
         *  must become active with LT(1) available.  consume() simply
         *  moves the input pointer so that LT(1) points at the next
         *  input symbol. Consume at least one token.
         *  </summary>
         *
         *  <remarks>
         *  Walk past any token not on the channel the parser is listening to.
         *  </remarks>
         */
        public virtual void Consume()
        {
            if ( p < tokens.Count )
            {
                p++;
                p = SkipOffTokenChannels( p ); // leave p on valid token
            }
        }

        /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */
        protected virtual int SkipOffTokenChannels( int i )
        {
            int n = tokens.Count;
            while ( i < n && ( (IToken)tokens[i] ).Channel != channel )
            {
                i++;
            }
            return i;
        }

        protected virtual int SkipOffTokenChannelsReverse( int i )
        {
            while ( i >= 0 && ( (IToken)tokens[i] ).Channel != channel )
            {
                i--;
            }
            return i;
        }

        /** <summary>
         *  A simple filter mechanism whereby you can tell this token stream
         *  to force all tokens of type ttype to be on channel.  For example,
         *  when interpreting, we cannot exec actions so we need to tell
         *  the stream to force all WS and NEWLINE to be a different, ignored
         *  channel.
         *  </summary>
         */
        public virtual void SetTokenTypeChannel( int ttype, int channel )
        {
            if ( channelOverrideMap == null )
            {
                channelOverrideMap = new Dictionary<int, int>();
            }
            channelOverrideMap[ttype] = channel;
        }

        public virtual void DiscardTokenType( int ttype )
        {
            if ( discardSet == null )
            {
                discardSet = new List<int>();
            }
            discardSet.Add( ttype );
        }

        public virtual void SetDiscardOffChannelTokens( bool discardOffChannelTokens )
        {
            this.discardOffChannelTokens = discardOffChannelTokens;
        }

        public virtual IList<IToken> GetTokens()
        {
            if ( p == -1 )
            {
                FillBuffer();
            }
            return tokens;
        }

        public virtual IList<IToken> GetTokens( int start, int stop )
        {
            return GetTokens( start, stop, (BitSet)null );
        }

        /** <summary>
         *  Given a start and stop index, return a List of all tokens in
         *  the token type BitSet.  Return null if no tokens were found.  This
         *  method looks at both on and off channel tokens.
         *  </summary>
         */
        public virtual IList<IToken> GetTokens( int start, int stop, BitSet types )
        {
            if ( p == -1 )
            {
                FillBuffer();
            }
            if ( stop >= tokens.Count )
            {
                stop = tokens.Count - 1;
            }
            if ( start < 0 )
            {
                start = 0;
            }
            if ( start > stop )
            {
                return null;
            }

            // list = tokens[start:stop]:{Token t, t.getType() in types}
            IList<IToken> filteredTokens = new List<IToken>();
            for ( int i = start; i <= stop; i++ )
            {
                IToken t = tokens[i];
                if ( types == null || types.Member( t.Type ) )
                {
                    filteredTokens.Add( t );
                }
            }
            if ( filteredTokens.Count == 0 )
            {
                filteredTokens = null;
            }
            return filteredTokens;
        }

        public virtual IList<IToken> GetTokens( int start, int stop, IList<int> types )
        {
            return GetTokens( start, stop, new BitSet( types ) );
        }

        public virtual IList<IToken> GetTokens( int start, int stop, int ttype )
        {
            return GetTokens( start, stop, BitSet.Of( ttype ) );
        }

        /** <summary>
         *  Get the ith token from the current position 1..n where k=1 is the
         *  first symbol of lookahead.
         *  </summary>
         */
        public virtual IToken LT( int k )
        {
            if ( p == -1 )
            {
                FillBuffer();
            }
            if ( k == 0 )
            {
                return null;
            }
            if ( k < 0 )
            {
                return LB( -k );
            }
            //System.out.print("LT(p="+p+","+k+")=");
            if ( ( p + k - 1 ) >= tokens.Count )
            {
                return tokens[tokens.Count - 1];
            }
            //System.out.println(tokens.get(p+k-1));
            int i = p;
            int n = 1;
            // find k good tokens
            while ( n < k )
            {
                // skip off-channel tokens
                i = SkipOffTokenChannels( i + 1 ); // leave p on valid token
                n++;
            }
            if ( i >= tokens.Count )
            {
                return tokens[tokens.Count - 1];
            }

            if (i > Range)
                Range = i;

            return (IToken)tokens[i];
        }

        /** <summary>Look backwards k tokens on-channel tokens</summary> */
        protected virtual IToken LB( int k )
        {
            //System.out.print("LB(p="+p+","+k+") ");
            if ( p == -1 )
            {
                FillBuffer();
            }
            if ( k == 0 )
            {
                return null;
            }
            if ( ( p - k ) < 0 )
            {
                return null;
            }

            int i = p;
            int n = 1;
            // find k good tokens looking backwards
            while ( n <= k )
            {
                // skip off-channel tokens
                i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token
                n++;
            }
            if ( i < 0 )
            {
                return null;
            }
            return (IToken)tokens[i];
        }

        /** <summary>
         *  Return absolute token i; ignore which channel the tokens are on;
         *  that is, count all tokens not just on-channel tokens.
         *  </summary>
         */
        public virtual IToken Get( int i )
        {
            return (IToken)tokens[i];
        }

#if false
        /** Get all tokens from start..stop inclusively */
        public virtual List<IToken> Get(int start, int count)
        {
            if (start < 0)
                throw new ArgumentOutOfRangeException("start");
            if (count < 0)
                throw new ArgumentOutOfRangeException("count");

            if (p == -1)
                FillBuffer();

            return new List<IToken>(tokens.Skip(start).Take(count));
        }
#endif

        public virtual int LA( int i )
        {
            return LT( i ).Type;
        }

        public virtual int Mark()
        {
            if ( p == -1 )
            {
                FillBuffer();
            }
            lastMarker = Index;
            return lastMarker;
        }

        public virtual void Release( int marker )
        {
            // no resources to release
        }

        public virtual int Count
        {
            get
            {
                return tokens.Count;
            }
        }

        public virtual void Rewind( int marker )
        {
            Seek( marker );
        }

        public virtual void Rewind()
        {
            Seek( lastMarker );
        }

        public virtual void Reset()
        {
            p = 0;
            lastMarker = 0;
        }

        public virtual void Seek( int index )
        {
            p = index;
        }

        public virtual ITokenSource TokenSource
        {
            get
            {
                return _tokenSource;
            }
        }

        public virtual string SourceName
        {
            get
            {
                return TokenSource.SourceName;
            }
        }

        public override string ToString()
        {
            if ( p == -1 )
            {
                throw new InvalidOperationException( "Buffer is not yet filled." );
            }
            return ToString( 0, tokens.Count - 1 );
        }

        public virtual string ToString( int start, int stop )
        {
            if ( start < 0 || stop < 0 )
            {
                return null;
            }
            if ( p == -1 )
            {
                throw new InvalidOperationException( "Buffer is not yet filled." );
            }
            if ( stop >= tokens.Count )
            {
                stop = tokens.Count - 1;
            }
            StringBuilder buf = new StringBuilder();
            for ( int i = start; i <= stop; i++ )
            {
                IToken t = tokens[i];
                buf.Append( t.Text );
            }
            return buf.ToString();
        }

        public virtual string ToString( IToken start, IToken stop )
        {
            if ( start != null && stop != null )
            {
                return ToString( start.TokenIndex, stop.TokenIndex );
            }
            return null;
        }
    }
}