src/antlr/TokenStreamRewriteEngine.hpp

#ifndef INC_TokenStreamRewriteEngine_hpp__
#define INC_TokenStreamRewriteEngine_hpp__

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 */

#include <string>
#include <list>
#include <vector>
#include <map>
#include <utility>
#include <iostream>
#include <iterator>
#include <cassert>
#include <algorithm>

#include <antlr/config.hpp>

#include <antlr/TokenStream.hpp>
#include <antlr/TokenWithIndex.hpp>
#include <antlr/BitSet.hpp>

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

/** This token stream tracks the *entire* token stream coming from
 *  a lexer, but does not pass on the whitespace (or whatever else
 *  you want to discard) to the parser.
 *
 *  This class can then be asked for the ith token in the input stream.
 *  Useful for dumping out the input stream exactly after doing some
 *  augmentation or other manipulations.  Tokens are index from 0..n-1
 *
 *  You can insert stuff, replace, and delete chunks.  Note that the
 *  operations are done lazily--only if you convert the buffer to a
 *  String.  This is very efficient because you are not moving data around
 *  all the time.  As the buffer of tokens is converted to strings, the
 *  toString() method(s) check to see if there is an operation at the
 *  current index.  If so, the operation is done and then normal String
 *  rendering continues on the buffer.  This is like having multiple Turing
 *  machine instruction streams (programs) operating on a single input tape. :)
 *
 *  Since the operations are done lazily at toString-time, operations do not
 *  screw up the token index values.  That is, an insert operation at token
 *  index i does not change the index values for tokens i+1..n-1.
 *
 *  Because operations never actually alter the buffer, you may always get
 *  the original token stream back without undoing anything.  Since
 *  the instructions are queued up, you can easily simulate transactions and
 *  roll back any changes if there is an error just by removing instructions.
 *  For example,
 *
 *              TokenStreamRewriteEngine rewriteEngine =
 *                      new TokenStreamRewriteEngine(lexer);
 *      JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
 *      ...
 *      rewriteEngine.insertAfter("pass1", t, "foobar");}
 *              rewriteEngine.insertAfter("pass2", u, "start");}
 *              System.out.println(rewriteEngine.toString("pass1"));
 *              System.out.println(rewriteEngine.toString("pass2"));
 *
 *  You can also have multiple "instruction streams" and get multiple
 *  rewrites from a single pass over the input.  Just name the instruction
 *  streams and use that name again when printing the buffer.  This could be
 *  useful for generating a C file and also its header file--all from the
 *  same buffer.
 *
 *  If you don't use named rewrite streams, a "default" stream is used.
 *
 *  Terence Parr, parrt@cs.usfca.edu
 *  University of San Francisco
 *  February 2004
 */
class TokenStreamRewriteEngine : public TokenStream
{
public:
        typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;

        static const size_t MIN_TOKEN_INDEX = 0;
        static const char* DEFAULT_PROGRAM_NAME;
        static const int PROGRAM_INIT_SIZE = 100;

        struct tokenToStream {
                tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
                template <typename T> void operator() ( const T& t ) {
                        out << t->getText();
                }
                ANTLR_USE_NAMESPACE(std)ostream& out;
        };

        class RewriteOperation {
        protected:
                RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
                : index(idx), text(txt)
                {
                }
        public:
                virtual ~RewriteOperation()
                {
                }
                /** Execute the rewrite operation by possibly adding to the buffer.
                 *  Return the index of the next token to operate on.
                 */
                virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
                        return index;
                }
                virtual size_t getIndex() const {
                        return index;
                }
                virtual const char* type() const {
                        return "RewriteOperation";
                }
        protected:
                size_t index;
                ANTLR_USE_NAMESPACE(std)string text;
        };

        struct executeOperation {
                ANTLR_USE_NAMESPACE(std)ostream& out;
                executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
                void operator () ( RewriteOperation* t ) {
                        t->execute(out);
                }
        };

        /// list of rewrite operations
        typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
        /// map program name to <program counter,program> tuple
        typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;

        class InsertBeforeOp : public RewriteOperation
        {
        public:
                InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
                : RewriteOperation(index, text)
                {
                }
                virtual ~InsertBeforeOp() {}
                virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
                {
                        out << text;
                        return index;
                }
                virtual const char* type() const {
                        return "InsertBeforeOp";
                }
        };

        class ReplaceOp : public RewriteOperation
        {
        public:
                ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
                : RewriteOperation(from,text)
                , lastIndex(to)
                {
                }
                virtual ~ReplaceOp() {}
                virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
                        out << text;
                        return lastIndex+1;
                }
                virtual const char* type() const {
                        return "ReplaceOp";
                }
        protected:
                size_t lastIndex;
        };

        class DeleteOp : public ReplaceOp {
        public:
                DeleteOp(size_t from, size_t to)
                : ReplaceOp(from,to,"")
                {
                }
                virtual const char* type() const {
                        return "DeleteOp";
                }
        };

        TokenStreamRewriteEngine(TokenStream& upstream);

        TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);

        RefToken nextToken( void );

        void rollback(size_t instructionIndex) {
                rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
        }

        /** Rollback the instruction stream for a program so that
         *  the indicated instruction (via instructionIndex) is no
         *  longer in the stream.  UNTESTED!
         */
        void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
                                          size_t instructionIndex );

        void deleteProgram() {
                deleteProgram(DEFAULT_PROGRAM_NAME);
        }

        /** Reset the program so that no instructions exist */
        void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
                rollback(programName, MIN_TOKEN_INDEX);
        }

        void insertAfter( RefTokenWithIndex t,
                                                        const ANTLR_USE_NAMESPACE(std)string& text )
        {
                insertAfter(DEFAULT_PROGRAM_NAME, t, text);
        }

        void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
                insertAfter(DEFAULT_PROGRAM_NAME, index, text);
        }

        void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                        RefTokenWithIndex t,
                                                        const ANTLR_USE_NAMESPACE(std)string& text )
        {
                insertAfter(programName, t->getIndex(), text);
        }

        void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                   size_t index,
                                                        const ANTLR_USE_NAMESPACE(std)string& text )
        {
                // to insert after, just insert before next index (even if past end)
                insertBefore(programName,index+1, text);
        }

        void insertBefore( RefTokenWithIndex t,
                                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                // std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
                insertBefore(DEFAULT_PROGRAM_NAME, t, text);
        }

        void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
                insertBefore(DEFAULT_PROGRAM_NAME, index, text);
        }

        void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                         RefTokenWithIndex t,
                                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                insertBefore(programName, t->getIndex(), text);
        }

        void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                         size_t index,
                                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
        }

        void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
        {
                replace(DEFAULT_PROGRAM_NAME, index, index, text);
        }

        void replace( size_t from, size_t to,
                                          const ANTLR_USE_NAMESPACE(std)string& text)
        {
                replace(DEFAULT_PROGRAM_NAME, from, to, text);
        }

        void replace( RefTokenWithIndex indexT,
                                          const ANTLR_USE_NAMESPACE(std)string& text )
        {
                replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
        }

        void replace( RefTokenWithIndex from,
                                          RefTokenWithIndex to,
                                          const ANTLR_USE_NAMESPACE(std)string& text )
        {
                replace(DEFAULT_PROGRAM_NAME, from, to, text);
        }

        void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
                                         size_t from, size_t to,
                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
        }

        void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
                                          RefTokenWithIndex from,
                                          RefTokenWithIndex to,
                                          const ANTLR_USE_NAMESPACE(std)string& text )
        {
                replace(programName,
                                  from->getIndex(),
                                  to->getIndex(),
                                  text);
        }

        void remove(size_t index) {
                remove(DEFAULT_PROGRAM_NAME, index, index);
        }

        void remove(size_t from, size_t to) {
                remove(DEFAULT_PROGRAM_NAME, from, to);
        }

        void remove(RefTokenWithIndex indexT) {
                remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
        }

        void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
                remove(DEFAULT_PROGRAM_NAME, from, to);
        }

        void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
                                         size_t from, size_t to)
        {
                replace(programName,from,to,"");
        }

        void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
                                         RefTokenWithIndex from, RefTokenWithIndex to )
        {
                replace(programName,from,to,"");
        }

        void discard(int ttype) {
                discardMask.add(ttype);
        }

        RefToken getToken( size_t i )
        {
                return RefToken(tokens.at(i));
        }

        size_t getTokenStreamSize() const {
                return tokens.size();
        }

        void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
                ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
        }

        void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                                  size_t start, size_t end ) const;

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
                return toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
        }

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                const ANTLR_USE_NAMESPACE(std)string& programName ) const
        {
                return toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
        }

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                size_t start, size_t end ) const
        {
                return toStream(out, DEFAULT_PROGRAM_NAME, start, end);
        }

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                const ANTLR_USE_NAMESPACE(std)string& programName,
                                                size_t firstToken, size_t lastToken ) const;

        void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
                return toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
        }

        void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                          size_t start, size_t end ) const;

        size_t getLastRewriteTokenIndex() const {
                return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
        }

        /** Return the last index for the program named programName
         * return 0 if the program does not exist or the program is empty.
         * (Note this is different from the java implementation that returns -1)
         */
        size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
                program_map::const_iterator rewrites = programs.find(programName);

                if( rewrites == programs.end() )
                        return 0;

                const operation_list& prog = rewrites->second;
                if( !prog.empty() )
                {
                        operation_list::const_iterator last = prog.end();
                        --last;
                        return (*last)->getIndex();
                }
                return 0;
        }

protected:
        /** If op.index > lastRewriteTokenIndexes, just add to the end.
         *  Otherwise, do linear */
        void addToSortedRewriteList(RewriteOperation* op) {
                addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
        }

        void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                                                  RewriteOperation* op );

protected:
        /** Who do we suck tokens from? */
        TokenStream& stream;
        /** track index of tokens */
        size_t index;

        /** Track the incoming list of tokens */
        token_list tokens;

        /** You may have multiple, named streams of rewrite operations.
         *  I'm calling these things "programs."
         *  Maps String (name) -> rewrite (List)
         */
        program_map programs;

        /** Which (whitespace) token(s) to throw out */
        BitSet discardMask;
};

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif

#endif
Revision:	2469
Committed:	Fri Dec 2 15:38:03 2005 UTC (18 years, 7 months ago) by tim
File size:	12539 byte(s)
Log Message:	End of the Link --> List Return of the Oject-Oriented replace yacc/lex parser with antlr parser
#	User	Rev	Content
1	tim	2469	#ifndef INC_TokenStreamRewriteEngine_hpp__
2			#define INC_TokenStreamRewriteEngine_hpp__
3
4			/* ANTLR Translator Generator
5			* Project led by Terence Parr at http://www.jGuru.com
6			* Software rights: http://www.antlr.org/license.html
7			*/
8
9			#include <string>
10			#include <list>
11			#include <vector>
12			#include <map>
13			#include <utility>
14			#include <iostream>
15			#include <iterator>
16			#include <cassert>
17			#include <algorithm>
18
19			#include <antlr/config.hpp>
20
21			#include <antlr/TokenStream.hpp>
22			#include <antlr/TokenWithIndex.hpp>
23			#include <antlr/BitSet.hpp>
24
25			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
26			namespace antlr {
27			#endif
28
29			/** This token stream tracks the entire token stream coming from
30			* a lexer, but does not pass on the whitespace (or whatever else
31			* you want to discard) to the parser.
32			*
33			* This class can then be asked for the ith token in the input stream.
34			* Useful for dumping out the input stream exactly after doing some
35			* augmentation or other manipulations. Tokens are index from 0..n-1
36			*
37			* You can insert stuff, replace, and delete chunks. Note that the
38			* operations are done lazily--only if you convert the buffer to a
39			* String. This is very efficient because you are not moving data around
40			* all the time. As the buffer of tokens is converted to strings, the
41			* toString() method(s) check to see if there is an operation at the
42			* current index. If so, the operation is done and then normal String
43			* rendering continues on the buffer. This is like having multiple Turing
44			* machine instruction streams (programs) operating on a single input tape. :)
45			*
46			* Since the operations are done lazily at toString-time, operations do not
47			* screw up the token index values. That is, an insert operation at token
48			* index i does not change the index values for tokens i+1..n-1.
49			*
50			* Because operations never actually alter the buffer, you may always get
51			* the original token stream back without undoing anything. Since
52			* the instructions are queued up, you can easily simulate transactions and
53			* roll back any changes if there is an error just by removing instructions.
54			* For example,
55			*
56			* TokenStreamRewriteEngine rewriteEngine =
57			* new TokenStreamRewriteEngine(lexer);
58			* JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
59			* ...
60			* rewriteEngine.insertAfter("pass1", t, "foobar");}
61			* rewriteEngine.insertAfter("pass2", u, "start");}
62			* System.out.println(rewriteEngine.toString("pass1"));
63			* System.out.println(rewriteEngine.toString("pass2"));
64			*
65			* You can also have multiple "instruction streams" and get multiple
66			* rewrites from a single pass over the input. Just name the instruction
67			* streams and use that name again when printing the buffer. This could be
68			* useful for generating a C file and also its header file--all from the
69			* same buffer.
70			*
71			* If you don't use named rewrite streams, a "default" stream is used.
72			*
73			* Terence Parr, parrt@cs.usfca.edu
74			* University of San Francisco
75			* February 2004
76			*/
77			class TokenStreamRewriteEngine : public TokenStream
78			{
79			public:
80			typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
81
82			static const size_t MIN_TOKEN_INDEX = 0;
83			static const char* DEFAULT_PROGRAM_NAME;
84			static const int PROGRAM_INIT_SIZE = 100;
85
86			struct tokenToStream {
87			tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
88			template <typename T> void operator() ( const T& t ) {
89			out << t->getText();
90			}
91			ANTLR_USE_NAMESPACE(std)ostream& out;
92			};
93
94			class RewriteOperation {
95			protected:
96			RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
97			: index(idx), text(txt)
98			{
99			}
100			public:
101			virtual ~RewriteOperation()
102			{
103			}
104			/** Execute the rewrite operation by possibly adding to the buffer.
105			* Return the index of the next token to operate on.
106			*/
107			virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
108			return index;
109			}
110			virtual size_t getIndex() const {
111			return index;
112			}
113			virtual const char* type() const {
114			return "RewriteOperation";
115			}
116			protected:
117			size_t index;
118			ANTLR_USE_NAMESPACE(std)string text;
119			};
120
121			struct executeOperation {
122			ANTLR_USE_NAMESPACE(std)ostream& out;
123			executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
124			void operator () ( RewriteOperation* t ) {
125			t->execute(out);
126			}
127			};
128
129			/// list of rewrite operations
130			typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
131			/// map program name to <program counter,program> tuple
132			typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;
133
134			class InsertBeforeOp : public RewriteOperation
135			{
136			public:
137			InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
138			: RewriteOperation(index, text)
139			{
140			}
141			virtual ~InsertBeforeOp() {}
142			virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
143			{
144			out << text;
145			return index;
146			}
147			virtual const char* type() const {
148			return "InsertBeforeOp";
149			}
150			};
151
152			class ReplaceOp : public RewriteOperation
153			{
154			public:
155			ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
156			: RewriteOperation(from,text)
157			, lastIndex(to)
158			{
159			}
160			virtual ~ReplaceOp() {}
161			virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
162			out << text;
163			return lastIndex+1;
164			}
165			virtual const char* type() const {
166			return "ReplaceOp";
167			}
168			protected:
169			size_t lastIndex;
170			};
171
172			class DeleteOp : public ReplaceOp {
173			public:
174			DeleteOp(size_t from, size_t to)
175			: ReplaceOp(from,to,"")
176			{
177			}
178			virtual const char* type() const {
179			return "DeleteOp";
180			}
181			};
182
183			TokenStreamRewriteEngine(TokenStream& upstream);
184
185			TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);
186
187			RefToken nextToken( void );
188
189			void rollback(size_t instructionIndex) {
190			rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
191			}
192
193			/** Rollback the instruction stream for a program so that
194			* the indicated instruction (via instructionIndex) is no
195			* longer in the stream. UNTESTED!
196			*/
197			void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
198			size_t instructionIndex );
199
200			void deleteProgram() {
201			deleteProgram(DEFAULT_PROGRAM_NAME);
202			}
203
204			/** Reset the program so that no instructions exist */
205			void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
206			rollback(programName, MIN_TOKEN_INDEX);
207			}
208
209			void insertAfter( RefTokenWithIndex t,
210			const ANTLR_USE_NAMESPACE(std)string& text )
211			{
212			insertAfter(DEFAULT_PROGRAM_NAME, t, text);
213			}
214
215			void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
216			insertAfter(DEFAULT_PROGRAM_NAME, index, text);
217			}
218
219			void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
220			RefTokenWithIndex t,
221			const ANTLR_USE_NAMESPACE(std)string& text )
222			{
223			insertAfter(programName, t->getIndex(), text);
224			}
225
226			void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
227			size_t index,
228			const ANTLR_USE_NAMESPACE(std)string& text )
229			{
230			// to insert after, just insert before next index (even if past end)
231			insertBefore(programName,index+1, text);
232			}
233
234			void insertBefore( RefTokenWithIndex t,
235			const ANTLR_USE_NAMESPACE(std)string& text )
236			{
237			// std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
238			insertBefore(DEFAULT_PROGRAM_NAME, t, text);
239			}
240
241			void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
242			insertBefore(DEFAULT_PROGRAM_NAME, index, text);
243			}
244
245			void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
246			RefTokenWithIndex t,
247			const ANTLR_USE_NAMESPACE(std)string& text )
248			{
249			insertBefore(programName, t->getIndex(), text);
250			}
251
252			void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
253			size_t index,
254			const ANTLR_USE_NAMESPACE(std)string& text )
255			{
256			addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
257			}
258
259			void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
260			{
261			replace(DEFAULT_PROGRAM_NAME, index, index, text);
262			}
263
264			void replace( size_t from, size_t to,
265			const ANTLR_USE_NAMESPACE(std)string& text)
266			{
267			replace(DEFAULT_PROGRAM_NAME, from, to, text);
268			}
269
270			void replace( RefTokenWithIndex indexT,
271			const ANTLR_USE_NAMESPACE(std)string& text )
272			{
273			replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
274			}
275
276			void replace( RefTokenWithIndex from,
277			RefTokenWithIndex to,
278			const ANTLR_USE_NAMESPACE(std)string& text )
279			{
280			replace(DEFAULT_PROGRAM_NAME, from, to, text);
281			}
282
283			void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
284			size_t from, size_t to,
285			const ANTLR_USE_NAMESPACE(std)string& text )
286			{
287			addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
288			}
289
290			void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
291			RefTokenWithIndex from,
292			RefTokenWithIndex to,
293			const ANTLR_USE_NAMESPACE(std)string& text )
294			{
295			replace(programName,
296			from->getIndex(),
297			to->getIndex(),
298			text);
299			}
300
301			void remove(size_t index) {
302			remove(DEFAULT_PROGRAM_NAME, index, index);
303			}
304
305			void remove(size_t from, size_t to) {
306			remove(DEFAULT_PROGRAM_NAME, from, to);
307			}
308
309			void remove(RefTokenWithIndex indexT) {
310			remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
311			}
312
313			void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
314			remove(DEFAULT_PROGRAM_NAME, from, to);
315			}
316
317			void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
318			size_t from, size_t to)
319			{
320			replace(programName,from,to,"");
321			}
322
323			void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
324			RefTokenWithIndex from, RefTokenWithIndex to )
325			{
326			replace(programName,from,to,"");
327			}
328
329			void discard(int ttype) {
330			discardMask.add(ttype);
331			}
332
333			RefToken getToken( size_t i )
334			{
335			return RefToken(tokens.at(i));
336			}
337
338			size_t getTokenStreamSize() const {
339			return tokens.size();
340			}
341
342			void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
343			ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
344			}
345
346			void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
347			size_t start, size_t end ) const;
348
349			void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
350			return toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
351			}
352
353			void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
354			const ANTLR_USE_NAMESPACE(std)string& programName ) const
355			{
356			return toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
357			}
358
359			void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
360			size_t start, size_t end ) const
361			{
362			return toStream(out, DEFAULT_PROGRAM_NAME, start, end);
363			}
364
365			void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
366			const ANTLR_USE_NAMESPACE(std)string& programName,
367			size_t firstToken, size_t lastToken ) const;
368
369			void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
370			return toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
371			}
372
373			void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
374			size_t start, size_t end ) const;
375
376			size_t getLastRewriteTokenIndex() const {
377			return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
378			}
379
380			/** Return the last index for the program named programName
381			* return 0 if the program does not exist or the program is empty.
382			* (Note this is different from the java implementation that returns -1)
383			*/
384			size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
385			program_map::const_iterator rewrites = programs.find(programName);
386
387			if( rewrites == programs.end() )
388			return 0;
389
390			const operation_list& prog = rewrites->second;
391			if( !prog.empty() )
392			{
393			operation_list::const_iterator last = prog.end();
394			--last;
395			return (*last)->getIndex();
396			}
397			return 0;
398			}
399
400			protected:
401			/** If op.index > lastRewriteTokenIndexes, just add to the end.
402			* Otherwise, do linear */
403			void addToSortedRewriteList(RewriteOperation* op) {
404			addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
405			}
406
407			void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
408			RewriteOperation* op );
409
410			protected:
411			/** Who do we suck tokens from? */
412			TokenStream& stream;
413			/** track index of tokens */
414			size_t index;
415
416			/** Track the incoming list of tokens */
417			token_list tokens;
418
419			/** You may have multiple, named streams of rewrite operations.
420			* I'm calling these things "programs."
421			* Maps String (name) -> rewrite (List)
422			*/
423			program_map programs;
424
425			/** Which (whitespace) token(s) to throw out */
426			BitSet discardMask;
427			};
428
429			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
430			}
431			#endif
432
433			#endif