src/antlr/TokenBuffer.hpp

#ifndef INC_TokenBuffer_hpp__
#define INC_TokenBuffer_hpp__

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 *
 * $Id: TokenBuffer.hpp,v 1.1 2005-12-02 15:38:02 tim Exp $
 */

#include <antlr/config.hpp>
#include <antlr/TokenStream.hpp>
#include <antlr/CircularQueue.hpp>

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

/**A Stream of Token objects fed to the parser from a TokenStream that can
 * be rewound via mark()/rewind() methods.
 * <p>
 * A dynamic array is used to buffer up all the input tokens.  Normally,
 * "k" tokens are stored in the buffer.  More tokens may be stored during
 * guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
 * Consumption of tokens is deferred.  In other words, reading the next
 * token is not done by conume(), but deferred until needed by LA or LT.
 * <p>
 *
 * @todo: see if we can integrate this one with InputBuffer into one template
 * or so.
 *
 * @see antlr.Token
 * @see antlr.TokenStream
 * @see antlr.TokenQueue
 */
class ANTLR_API TokenBuffer {
public:
        /** Create a token buffer */
        TokenBuffer(TokenStream& input_);
        virtual ~TokenBuffer();

        /// Reset the input buffer to empty state
        inline void reset( void )
        {
                nMarkers = 0;
                markerOffset = 0;
                numToConsume = 0;
                queue.clear();
        }

        /** Get a lookahead token value */
        int LA( unsigned int i );

        /** Get a lookahead token */
        RefToken LT( unsigned int i );

        /** Return an integer marker that can be used to rewind the buffer to
         * its current state.
         */
        unsigned int mark();

        /**Rewind the token buffer to a marker.
         * @param mark Marker returned previously from mark()
         */
        void rewind(unsigned int mark);

        /** Mark another token for deferred consumption */
        inline void consume()
        {
                numToConsume++;
        }

        /// Return the number of entries in the TokenBuffer
        virtual unsigned int entries() const;

private:
        /** Ensure that the token buffer is sufficiently full */
        void fill(unsigned int amount);
        /** Sync up deferred consumption */
        void syncConsume();

protected:
        /// Token source
        TokenStream& input;

        /// Number of active markers
        unsigned int nMarkers;

        /// Additional offset used when markers are active
        unsigned int markerOffset;

        /// Number of calls to consume() since last LA() or LT() call
        unsigned int numToConsume;

        /// Circular queue with Tokens
        CircularQueue<RefToken> queue;

private:
        TokenBuffer(const TokenBuffer& other);
        const TokenBuffer& operator=(const TokenBuffer& other);
};

/** Sync up deferred consumption */
inline void TokenBuffer::syncConsume()
{
        if (numToConsume > 0)
        {
                if (nMarkers > 0)
                        markerOffset += numToConsume;
                else
                        queue.removeItems( numToConsume );

                numToConsume = 0;
        }
}

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif

#endif //INC_TokenBuffer_hpp__
Revision:	2469
Committed:	Fri Dec 2 15:38:03 2005 UTC (18 years, 7 months ago) by tim
File size:	2870 byte(s)
Log Message:	End of the Link --> List Return of the Oject-Oriented replace yacc/lex parser with antlr parser
#	User	Rev	Content
1	tim	2469	#ifndef INC_TokenBuffer_hpp__
2			#define INC_TokenBuffer_hpp__
3
4			/* ANTLR Translator Generator
5			* Project led by Terence Parr at http://www.jGuru.com
6			* Software rights: http://www.antlr.org/license.html
7			*
8			* $Id: TokenBuffer.hpp,v 1.1 2005-12-02 15:38:02 tim Exp $
9			*/
10
11			#include <antlr/config.hpp>
12			#include <antlr/TokenStream.hpp>
13			#include <antlr/CircularQueue.hpp>
14
15			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
16			namespace antlr {
17			#endif
18
19			/**A Stream of Token objects fed to the parser from a TokenStream that can
20			* be rewound via mark()/rewind() methods.
21			* <p>
22			* A dynamic array is used to buffer up all the input tokens. Normally,
23			* "k" tokens are stored in the buffer. More tokens may be stored during
24			* guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
25			* Consumption of tokens is deferred. In other words, reading the next
26			* token is not done by conume(), but deferred until needed by LA or LT.
27			* <p>
28			*
29			* @todo: see if we can integrate this one with InputBuffer into one template
30			* or so.
31			*
32			* @see antlr.Token
33			* @see antlr.TokenStream
34			* @see antlr.TokenQueue
35			*/
36			class ANTLR_API TokenBuffer {
37			public:
38			/** Create a token buffer */
39			TokenBuffer(TokenStream& input_);
40			virtual ~TokenBuffer();
41
42			/// Reset the input buffer to empty state
43			inline void reset( void )
44			{
45			nMarkers = 0;
46			markerOffset = 0;
47			numToConsume = 0;
48			queue.clear();
49			}
50
51			/** Get a lookahead token value */
52			int LA( unsigned int i );
53
54			/** Get a lookahead token */
55			RefToken LT( unsigned int i );
56
57			/** Return an integer marker that can be used to rewind the buffer to
58			* its current state.
59			*/
60			unsigned int mark();
61
62			/**Rewind the token buffer to a marker.
63			* @param mark Marker returned previously from mark()
64			*/
65			void rewind(unsigned int mark);
66
67			/** Mark another token for deferred consumption */
68			inline void consume()
69			{
70			numToConsume++;
71			}
72
73			/// Return the number of entries in the TokenBuffer
74			virtual unsigned int entries() const;
75
76			private:
77			/** Ensure that the token buffer is sufficiently full */
78			void fill(unsigned int amount);
79			/** Sync up deferred consumption */
80			void syncConsume();
81
82			protected:
83			/// Token source
84			TokenStream& input;
85
86			/// Number of active markers
87			unsigned int nMarkers;
88
89			/// Additional offset used when markers are active
90			unsigned int markerOffset;
91
92			/// Number of calls to consume() since last LA() or LT() call
93			unsigned int numToConsume;
94
95			/// Circular queue with Tokens
96			CircularQueue<RefToken> queue;
97
98			private:
99			TokenBuffer(const TokenBuffer& other);
100			const TokenBuffer& operator=(const TokenBuffer& other);
101			};
102
103			/** Sync up deferred consumption */
104			inline void TokenBuffer::syncConsume()
105			{
106			if (numToConsume > 0)
107			{
108			if (nMarkers > 0)
109			markerOffset += numToConsume;
110			else
111			queue.removeItems( numToConsume );
112
113			numToConsume = 0;
114			}
115			}
116
117			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
118			}
119			#endif
120
121			#endif //INC_TokenBuffer_hpp__