src/antlr/TokenStreamRewriteEngine.hpp

#ifndef INC_TokenStreamRewriteEngine_hpp__
#define INC_TokenStreamRewriteEngine_hpp__

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 */

#include <string>
#include <list>
#include <vector>
#include <map>
#include <utility>
#include <iostream>
#include <iterator>
#include <cassert>
#include <algorithm>

#include <antlr/config.hpp>

#include <antlr/TokenStream.hpp>
#include <antlr/TokenWithIndex.hpp>
#include <antlr/BitSet.hpp>

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

/** This token stream tracks the *entire* token stream coming from
 *       a lexer, but does not pass on the whitespace (or whatever else
 *       you want to discard) to the parser.
 *
 *       This class can then be asked for the ith token in the input stream.
 *       Useful for dumping out the input stream exactly after doing some
 *       augmentation or other manipulations.   Tokens are index from 0..n-1
 *
 *       You can insert stuff, replace, and delete chunks.       Note that the
 *       operations are done lazily--only if you convert the buffer to a
 *       String.         This is very efficient because you are not moving data around
 *       all the time.   As the buffer of tokens is converted to strings, the
 *       toString() method(s) check to see if there is an operation at the
 *       current index.  If so, the operation is done and then normal String
 *       rendering continues on the buffer.      This is like having multiple Turing
 *       machine instruction streams (programs) operating on a single input tape. :)
 *
 *       Since the operations are done lazily at toString-time, operations do not
 *       screw up the token index values.  That is, an insert operation at token
 *       index i does not change the index values for tokens i+1..n-1.
 *
 *       Because operations never actually alter the buffer, you may always get
 *       the original token stream back without undoing anything.  Since
 *       the instructions are queued up, you can easily simulate transactions and
 *       roll back any changes if there is an error just by removing instructions.
 *       For example,
 *
 *                      TokenStreamRewriteEngine rewriteEngine =
 *                              new TokenStreamRewriteEngine(lexer);
 *                JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
 *                ...
 *                rewriteEngine.insertAfter("pass1", t, "foobar");}
 *                      rewriteEngine.insertAfter("pass2", u, "start");}
 *                      System.out.println(rewriteEngine.toString("pass1"));
 *                      System.out.println(rewriteEngine.toString("pass2"));
 *
 *       You can also have multiple "instruction streams" and get multiple
 *       rewrites from a single pass over the input.     Just name the instruction
 *       streams and use that name again when printing the buffer.      This could be
 *       useful for generating a C file and also its header file--all from the
 *       same buffer.
 *
 *       If you don't use named rewrite streams, a "default" stream is used.
 *
 *       Terence Parr, parrt@cs.usfca.edu
 *       University of San Francisco
 *       February 2004
 */
class TokenStreamRewriteEngine : public TokenStream
{
public:
        typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
        static const char* DEFAULT_PROGRAM_NAME;
#ifndef NO_STATIC_CONSTS
        static const size_t MIN_TOKEN_INDEX;
        static const int PROGRAM_INIT_SIZE;
#else
        enum {
                MIN_TOKEN_INDEX = 0,
                PROGRAM_INIT_SIZE = 100
        };
#endif

        struct tokenToStream {
                tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
                template <typename T> void operator() ( const T& t ) {
                        out << t->getText();
                }
                ANTLR_USE_NAMESPACE(std)ostream& out;
        };

        class RewriteOperation {
        protected:
                RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
                : index(idx), text(txt)
                {
                }
        public:
                virtual ~RewriteOperation()
                {
                }
                /** Execute the rewrite operation by possibly adding to the buffer.
                 *       Return the index of the next token to operate on.
                 */
                virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
                        return index;
                }
                virtual size_t getIndex() const {
                        return index;
                }
                virtual const char* type() const {
                        return "RewriteOperation";
                }
        protected:
                size_t index;
                ANTLR_USE_NAMESPACE(std)string text;
        };

        struct executeOperation {
                ANTLR_USE_NAMESPACE(std)ostream& out;
                executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
                void operator () ( RewriteOperation* t ) {
                        t->execute(out);
                }
        };

        /// list of rewrite operations
        typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
        /// map program name to <program counter,program> tuple
        typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;

        class InsertBeforeOp : public RewriteOperation
        {
        public:
                InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
                : RewriteOperation(index, text)
                {
                }
                virtual ~InsertBeforeOp() {}
                virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
                {
                        out << text;
                        return index;
                }
                virtual const char* type() const {
                        return "InsertBeforeOp";
                }
        };

        class ReplaceOp : public RewriteOperation
        {
        public:
                ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
                : RewriteOperation(from,text)
                , lastIndex(to)
                {
                }
                virtual ~ReplaceOp() {}
                virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
                        out << text;
                        return lastIndex+1;
                }
                virtual const char* type() const {
                        return "ReplaceOp";
                }
        protected:
                size_t lastIndex;
        };

        class DeleteOp : public ReplaceOp {
        public:
                DeleteOp(size_t from, size_t to)
                : ReplaceOp(from,to,"")
                {
                }
                virtual const char* type() const {
                        return "DeleteOp";
                }
        };

        TokenStreamRewriteEngine(TokenStream& upstream);

        TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);

        RefToken nextToken( void );

        void rollback(size_t instructionIndex) {
                rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
        }

        /** Rollback the instruction stream for a program so that
         *       the indicated instruction (via instructionIndex) is no
         *       longer in the stream.  UNTESTED!
         */
        void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
                                          size_t instructionIndex );

        void deleteProgram() {
                deleteProgram(DEFAULT_PROGRAM_NAME);
        }

        /** Reset the program so that no instructions exist */
        void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
                rollback(programName, MIN_TOKEN_INDEX);
        }

        void insertAfter( RefTokenWithIndex t,
                                                        const ANTLR_USE_NAMESPACE(std)string& text )
        {
                insertAfter(DEFAULT_PROGRAM_NAME, t, text);
        }

        void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
                insertAfter(DEFAULT_PROGRAM_NAME, index, text);
        }

        void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                        RefTokenWithIndex t,
                                                        const ANTLR_USE_NAMESPACE(std)string& text )
        {
                insertAfter(programName, t->getIndex(), text);
        }

        void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                        size_t index,
                                                        const ANTLR_USE_NAMESPACE(std)string& text )
        {
                // to insert after, just insert before next index (even if past end)
                insertBefore(programName,index+1, text);
        }

        void insertBefore( RefTokenWithIndex t,
                                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                // std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
                insertBefore(DEFAULT_PROGRAM_NAME, t, text);
        }

        void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
                insertBefore(DEFAULT_PROGRAM_NAME, index, text);
        }

        void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                         RefTokenWithIndex t,
                                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                insertBefore(programName, t->getIndex(), text);
        }

        void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                         size_t index,
                                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
        }

        void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
        {
                replace(DEFAULT_PROGRAM_NAME, index, index, text);
        }

        void replace( size_t from, size_t to,
                                          const ANTLR_USE_NAMESPACE(std)string& text)
        {
                replace(DEFAULT_PROGRAM_NAME, from, to, text);
        }

        void replace( RefTokenWithIndex indexT,
                                          const ANTLR_USE_NAMESPACE(std)string& text )
        {
                replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
        }

        void replace( RefTokenWithIndex from,
                                          RefTokenWithIndex to,
                                          const ANTLR_USE_NAMESPACE(std)string& text )
        {
                replace(DEFAULT_PROGRAM_NAME, from, to, text);
        }

        void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
                                         size_t from, size_t to,
                                         const ANTLR_USE_NAMESPACE(std)string& text )
        {
                addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
        }

        void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
                                          RefTokenWithIndex from,
                                          RefTokenWithIndex to,
                                          const ANTLR_USE_NAMESPACE(std)string& text )
        {
                replace(programName,
                                  from->getIndex(),
                                  to->getIndex(),
                                  text);
        }

        void remove(size_t index) {
                remove(DEFAULT_PROGRAM_NAME, index, index);
        }

        void remove(size_t from, size_t to) {
                remove(DEFAULT_PROGRAM_NAME, from, to);
        }

        void remove(RefTokenWithIndex indexT) {
                remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
        }

        void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
                remove(DEFAULT_PROGRAM_NAME, from, to);
        }

        void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
                                         size_t from, size_t to)
        {
                replace(programName,from,to,"");
        }

        void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
                                         RefTokenWithIndex from, RefTokenWithIndex to )
        {
                replace(programName,from,to,"");
        }

        void discard(int ttype) {
                discardMask.add(ttype);
        }

        RefToken getToken( size_t i )
        {
                return RefToken(tokens.at(i));
        }

        size_t getTokenStreamSize() const {
                return tokens.size();
        }

        void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
                ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
        }

        void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                                  size_t start, size_t end ) const;

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
                toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
        }

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                const ANTLR_USE_NAMESPACE(std)string& programName ) const
        {
                toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
        }

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                size_t start, size_t end ) const
        {
                toStream(out, DEFAULT_PROGRAM_NAME, start, end);
        }

        void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                const ANTLR_USE_NAMESPACE(std)string& programName,
                                                size_t firstToken, size_t lastToken ) const;

        void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
                toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
        }

        void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
                                                          size_t start, size_t end ) const;

        size_t getLastRewriteTokenIndex() const {
                return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
        }

        /** Return the last index for the program named programName
         * return 0 if the program does not exist or the program is empty.
         * (Note this is different from the java implementation that returns -1)
         */
        size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
                program_map::const_iterator rewrites = programs.find(programName);

                if( rewrites == programs.end() )
                        return 0;

                const operation_list& prog = rewrites->second;
                if( !prog.empty() )
                {
                        operation_list::const_iterator last = prog.end();
                        --last;
                        return (*last)->getIndex();
                }
                return 0;
        }

protected:
        /** If op.index > lastRewriteTokenIndexes, just add to the end.
         *       Otherwise, do linear */
        void addToSortedRewriteList(RewriteOperation* op) {
                addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
        }

        void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
                                                                                  RewriteOperation* op );

protected:
        /** Who do we suck tokens from? */
        TokenStream& stream;
        /** track index of tokens */
        size_t index;

        /** Track the incoming list of tokens */
        token_list tokens;

        /** You may have multiple, named streams of rewrite operations.
         *  I'm calling these things "programs."
         *  Maps String (name) -> rewrite (List)
         */
        program_map programs;

        /** Which (whitespace) token(s) to throw out */
        BitSet discardMask;
};

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif

#endif
Revision:	1558
Committed:	Wed May 11 16:32:48 2011 UTC (14 years, 5 months ago) by gezelter
File size:	12588 byte(s)
Log Message:	Updated antlr, some minor formatting changes
#	Content
1	#ifndef INC_TokenStreamRewriteEngine_hpp__
2	#define INC_TokenStreamRewriteEngine_hpp__
3
4	/* ANTLR Translator Generator
5	* Project led by Terence Parr at http://www.jGuru.com
6	* Software rights: http://www.antlr.org/license.html
7	*/
8
9	#include <string>
10	#include <list>
11	#include <vector>
12	#include <map>
13	#include <utility>
14	#include <iostream>
15	#include <iterator>
16	#include <cassert>
17	#include <algorithm>
18
19	#include <antlr/config.hpp>
20
21	#include <antlr/TokenStream.hpp>
22	#include <antlr/TokenWithIndex.hpp>
23	#include <antlr/BitSet.hpp>
24
25	#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
26	namespace antlr {
27	#endif
28
29	/** This token stream tracks the entire token stream coming from
30	* a lexer, but does not pass on the whitespace (or whatever else
31	* you want to discard) to the parser.
32	*
33	* This class can then be asked for the ith token in the input stream.
34	* Useful for dumping out the input stream exactly after doing some
35	* augmentation or other manipulations. Tokens are index from 0..n-1
36	*
37	* You can insert stuff, replace, and delete chunks. Note that the
38	* operations are done lazily--only if you convert the buffer to a
39	* String. This is very efficient because you are not moving data around
40	* all the time. As the buffer of tokens is converted to strings, the
41	* toString() method(s) check to see if there is an operation at the
42	* current index. If so, the operation is done and then normal String
43	* rendering continues on the buffer. This is like having multiple Turing
44	* machine instruction streams (programs) operating on a single input tape. :)
45	*
46	* Since the operations are done lazily at toString-time, operations do not
47	* screw up the token index values. That is, an insert operation at token
48	* index i does not change the index values for tokens i+1..n-1.
49	*
50	* Because operations never actually alter the buffer, you may always get
51	* the original token stream back without undoing anything. Since
52	* the instructions are queued up, you can easily simulate transactions and
53	* roll back any changes if there is an error just by removing instructions.
54	* For example,
55	*
56	* TokenStreamRewriteEngine rewriteEngine =
57	* new TokenStreamRewriteEngine(lexer);
58	* JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
59	* ...
60	* rewriteEngine.insertAfter("pass1", t, "foobar");}
61	* rewriteEngine.insertAfter("pass2", u, "start");}
62	* System.out.println(rewriteEngine.toString("pass1"));
63	* System.out.println(rewriteEngine.toString("pass2"));
64	*
65	* You can also have multiple "instruction streams" and get multiple
66	* rewrites from a single pass over the input. Just name the instruction
67	* streams and use that name again when printing the buffer. This could be
68	* useful for generating a C file and also its header file--all from the
69	* same buffer.
70	*
71	* If you don't use named rewrite streams, a "default" stream is used.
72	*
73	* Terence Parr, parrt@cs.usfca.edu
74	* University of San Francisco
75	* February 2004
76	*/
77	class TokenStreamRewriteEngine : public TokenStream
78	{
79	public:
80	typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
81	static const char* DEFAULT_PROGRAM_NAME;
82	#ifndef NO_STATIC_CONSTS
83	static const size_t MIN_TOKEN_INDEX;
84	static const int PROGRAM_INIT_SIZE;
85	#else
86	enum {
87	MIN_TOKEN_INDEX = 0,
88	PROGRAM_INIT_SIZE = 100
89	};
90	#endif
91
92	struct tokenToStream {
93	tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
94	template <typename T> void operator() ( const T& t ) {
95	out << t->getText();
96	}
97	ANTLR_USE_NAMESPACE(std)ostream& out;
98	};
99
100	class RewriteOperation {
101	protected:
102	RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
103	: index(idx), text(txt)
104	{
105	}
106	public:
107	virtual ~RewriteOperation()
108	{
109	}
110	/** Execute the rewrite operation by possibly adding to the buffer.
111	* Return the index of the next token to operate on.
112	*/
113	virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
114	return index;
115	}
116	virtual size_t getIndex() const {
117	return index;
118	}
119	virtual const char* type() const {
120	return "RewriteOperation";
121	}
122	protected:
123	size_t index;
124	ANTLR_USE_NAMESPACE(std)string text;
125	};
126
127	struct executeOperation {
128	ANTLR_USE_NAMESPACE(std)ostream& out;
129	executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
130	void operator () ( RewriteOperation* t ) {
131	t->execute(out);
132	}
133	};
134
135	/// list of rewrite operations
136	typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
137	/// map program name to <program counter,program> tuple
138	typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;
139
140	class InsertBeforeOp : public RewriteOperation
141	{
142	public:
143	InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
144	: RewriteOperation(index, text)
145	{
146	}
147	virtual ~InsertBeforeOp() {}
148	virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
149	{
150	out << text;
151	return index;
152	}
153	virtual const char* type() const {
154	return "InsertBeforeOp";
155	}
156	};
157
158	class ReplaceOp : public RewriteOperation
159	{
160	public:
161	ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
162	: RewriteOperation(from,text)
163	, lastIndex(to)
164	{
165	}
166	virtual ~ReplaceOp() {}
167	virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
168	out << text;
169	return lastIndex+1;
170	}
171	virtual const char* type() const {
172	return "ReplaceOp";
173	}
174	protected:
175	size_t lastIndex;
176	};
177
178	class DeleteOp : public ReplaceOp {
179	public:
180	DeleteOp(size_t from, size_t to)
181	: ReplaceOp(from,to,"")
182	{
183	}
184	virtual const char* type() const {
185	return "DeleteOp";
186	}
187	};
188
189	TokenStreamRewriteEngine(TokenStream& upstream);
190
191	TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);
192
193	RefToken nextToken( void );
194
195	void rollback(size_t instructionIndex) {
196	rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
197	}
198
199	/** Rollback the instruction stream for a program so that
200	* the indicated instruction (via instructionIndex) is no
201	* longer in the stream. UNTESTED!
202	*/
203	void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
204	size_t instructionIndex );
205
206	void deleteProgram() {
207	deleteProgram(DEFAULT_PROGRAM_NAME);
208	}
209
210	/** Reset the program so that no instructions exist */
211	void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
212	rollback(programName, MIN_TOKEN_INDEX);
213	}
214
215	void insertAfter( RefTokenWithIndex t,
216	const ANTLR_USE_NAMESPACE(std)string& text )
217	{
218	insertAfter(DEFAULT_PROGRAM_NAME, t, text);
219	}
220
221	void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
222	insertAfter(DEFAULT_PROGRAM_NAME, index, text);
223	}
224
225	void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
226	RefTokenWithIndex t,
227	const ANTLR_USE_NAMESPACE(std)string& text )
228	{
229	insertAfter(programName, t->getIndex(), text);
230	}
231
232	void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
233	size_t index,
234	const ANTLR_USE_NAMESPACE(std)string& text )
235	{
236	// to insert after, just insert before next index (even if past end)
237	insertBefore(programName,index+1, text);
238	}
239
240	void insertBefore( RefTokenWithIndex t,
241	const ANTLR_USE_NAMESPACE(std)string& text )
242	{
243	// std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
244	insertBefore(DEFAULT_PROGRAM_NAME, t, text);
245	}
246
247	void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
248	insertBefore(DEFAULT_PROGRAM_NAME, index, text);
249	}
250
251	void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
252	RefTokenWithIndex t,
253	const ANTLR_USE_NAMESPACE(std)string& text )
254	{
255	insertBefore(programName, t->getIndex(), text);
256	}
257
258	void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
259	size_t index,
260	const ANTLR_USE_NAMESPACE(std)string& text )
261	{
262	addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
263	}
264
265	void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
266	{
267	replace(DEFAULT_PROGRAM_NAME, index, index, text);
268	}
269
270	void replace( size_t from, size_t to,
271	const ANTLR_USE_NAMESPACE(std)string& text)
272	{
273	replace(DEFAULT_PROGRAM_NAME, from, to, text);
274	}
275
276	void replace( RefTokenWithIndex indexT,
277	const ANTLR_USE_NAMESPACE(std)string& text )
278	{
279	replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
280	}
281
282	void replace( RefTokenWithIndex from,
283	RefTokenWithIndex to,
284	const ANTLR_USE_NAMESPACE(std)string& text )
285	{
286	replace(DEFAULT_PROGRAM_NAME, from, to, text);
287	}
288
289	void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
290	size_t from, size_t to,
291	const ANTLR_USE_NAMESPACE(std)string& text )
292	{
293	addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
294	}
295
296	void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
297	RefTokenWithIndex from,
298	RefTokenWithIndex to,
299	const ANTLR_USE_NAMESPACE(std)string& text )
300	{
301	replace(programName,
302	from->getIndex(),
303	to->getIndex(),
304	text);
305	}
306
307	void remove(size_t index) {
308	remove(DEFAULT_PROGRAM_NAME, index, index);
309	}
310
311	void remove(size_t from, size_t to) {
312	remove(DEFAULT_PROGRAM_NAME, from, to);
313	}
314
315	void remove(RefTokenWithIndex indexT) {
316	remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
317	}
318
319	void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
320	remove(DEFAULT_PROGRAM_NAME, from, to);
321	}
322
323	void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
324	size_t from, size_t to)
325	{
326	replace(programName,from,to,"");
327	}
328
329	void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
330	RefTokenWithIndex from, RefTokenWithIndex to )
331	{
332	replace(programName,from,to,"");
333	}
334
335	void discard(int ttype) {
336	discardMask.add(ttype);
337	}
338
339	RefToken getToken( size_t i )
340	{
341	return RefToken(tokens.at(i));
342	}
343
344	size_t getTokenStreamSize() const {
345	return tokens.size();
346	}
347
348	void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
349	ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
350	}
351
352	void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
353	size_t start, size_t end ) const;
354
355	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
356	toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
357	}
358
359	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
360	const ANTLR_USE_NAMESPACE(std)string& programName ) const
361	{
362	toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
363	}
364
365	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
366	size_t start, size_t end ) const
367	{
368	toStream(out, DEFAULT_PROGRAM_NAME, start, end);
369	}
370
371	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
372	const ANTLR_USE_NAMESPACE(std)string& programName,
373	size_t firstToken, size_t lastToken ) const;
374
375	void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
376	toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
377	}
378
379	void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
380	size_t start, size_t end ) const;
381
382	size_t getLastRewriteTokenIndex() const {
383	return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
384	}
385
386	/** Return the last index for the program named programName
387	* return 0 if the program does not exist or the program is empty.
388	* (Note this is different from the java implementation that returns -1)
389	*/
390	size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
391	program_map::const_iterator rewrites = programs.find(programName);
392
393	if( rewrites == programs.end() )
394	return 0;
395
396	const operation_list& prog = rewrites->second;
397	if( !prog.empty() )
398	{
399	operation_list::const_iterator last = prog.end();
400	--last;
401	return (*last)->getIndex();
402	}
403	return 0;
404	}
405
406	protected:
407	/** If op.index > lastRewriteTokenIndexes, just add to the end.
408	* Otherwise, do linear */
409	void addToSortedRewriteList(RewriteOperation* op) {
410	addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
411	}
412
413	void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
414	RewriteOperation* op );
415
416	protected:
417	/** Who do we suck tokens from? */
418	TokenStream& stream;
419	/** track index of tokens */
420	size_t index;
421
422	/** Track the incoming list of tokens */
423	token_list tokens;
424
425	/** You may have multiple, named streams of rewrite operations.
426	* I'm calling these things "programs."
427	* Maps String (name) -> rewrite (List)
428	*/
429	program_map programs;
430
431	/** Which (whitespace) token(s) to throw out */
432	BitSet discardMask;
433	};
434
435	#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
436	}
437	#endif
438
439	#endif