ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/OpenMD/branches/development/src/antlr/TokenStreamRewriteEngine.hpp
(Generate patch)

Comparing branches/development/src/antlr/TokenStreamRewriteEngine.hpp (file contents):
Revision 1465 by chuckv, Fri Jul 9 23:08:25 2010 UTC vs.
Revision 1633 by gezelter, Thu Sep 15 13:39:36 2011 UTC

# Line 27 | Line 27 | namespace antlr {
27   #endif
28  
29   /** This token stream tracks the *entire* token stream coming from
30 < *  a lexer, but does not pass on the whitespace (or whatever else
31 < *  you want to discard) to the parser.
30 > *       a lexer, but does not pass on the whitespace (or whatever else
31 > *       you want to discard) to the parser.
32   *
33 < *  This class can then be asked for the ith token in the input stream.
34 < *  Useful for dumping out the input stream exactly after doing some
35 < *  augmentation or other manipulations.  Tokens are index from 0..n-1
33 > *       This class can then be asked for the ith token in the input stream.
34 > *       Useful for dumping out the input stream exactly after doing some
35 > *       augmentation or other manipulations.   Tokens are index from 0..n-1
36   *
37 < *  You can insert stuff, replace, and delete chunks.  Note that the
38 < *  operations are done lazily--only if you convert the buffer to a
39 < *  String.  This is very efficient because you are not moving data around
40 < *  all the time.  As the buffer of tokens is converted to strings, the
41 < *  toString() method(s) check to see if there is an operation at the
42 < *  current index.  If so, the operation is done and then normal String
43 < *  rendering continues on the buffer.  This is like having multiple Turing
44 < *  machine instruction streams (programs) operating on a single input tape. :)
37 > *       You can insert stuff, replace, and delete chunks.       Note that the
38 > *       operations are done lazily--only if you convert the buffer to a
39 > *       String.         This is very efficient because you are not moving data around
40 > *       all the time.   As the buffer of tokens is converted to strings, the
41 > *       toString() method(s) check to see if there is an operation at the
42 > *       current index.  If so, the operation is done and then normal String
43 > *       rendering continues on the buffer.      This is like having multiple Turing
44 > *       machine instruction streams (programs) operating on a single input tape. :)
45   *
46 < *  Since the operations are done lazily at toString-time, operations do not
47 < *  screw up the token index values.  That is, an insert operation at token
48 < *  index i does not change the index values for tokens i+1..n-1.
46 > *       Since the operations are done lazily at toString-time, operations do not
47 > *       screw up the token index values.  That is, an insert operation at token
48 > *       index i does not change the index values for tokens i+1..n-1.
49   *
50 < *  Because operations never actually alter the buffer, you may always get
51 < *  the original token stream back without undoing anything.  Since
52 < *  the instructions are queued up, you can easily simulate transactions and
53 < *  roll back any changes if there is an error just by removing instructions.
54 < *  For example,
50 > *       Because operations never actually alter the buffer, you may always get
51 > *       the original token stream back without undoing anything.  Since
52 > *       the instructions are queued up, you can easily simulate transactions and
53 > *       roll back any changes if there is an error just by removing instructions.
54 > *       For example,
55   *
56 < *              TokenStreamRewriteEngine rewriteEngine =
57 < *                      new TokenStreamRewriteEngine(lexer);
58 < *      JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
59 < *      ...
60 < *      rewriteEngine.insertAfter("pass1", t, "foobar");}
61 < *              rewriteEngine.insertAfter("pass2", u, "start");}
62 < *              System.out.println(rewriteEngine.toString("pass1"));
63 < *              System.out.println(rewriteEngine.toString("pass2"));
64 < *
65 < *  You can also have multiple "instruction streams" and get multiple
66 < *  rewrites from a single pass over the input.  Just name the instruction
67 < *  streams and use that name again when printing the buffer.  This could be
68 < *  useful for generating a C file and also its header file--all from the
69 < *  same buffer.
56 > *                      TokenStreamRewriteEngine rewriteEngine =
57 > *                              new TokenStreamRewriteEngine(lexer);
58 > *                JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
59 > *                ...
60 > *                rewriteEngine.insertAfter("pass1", t, "foobar");}
61 > *                      rewriteEngine.insertAfter("pass2", u, "start");}
62 > *                      System.out.println(rewriteEngine.toString("pass1"));
63 > *                      System.out.println(rewriteEngine.toString("pass2"));
64   *
65 < *  If you don't use named rewrite streams, a "default" stream is used.
65 > *       You can also have multiple "instruction streams" and get multiple
66 > *       rewrites from a single pass over the input.     Just name the instruction
67 > *       streams and use that name again when printing the buffer.      This could be
68 > *       useful for generating a C file and also its header file--all from the
69 > *       same buffer.
70   *
71 < *  Terence Parr, parrt@cs.usfca.edu
72 < *  University of San Francisco
73 < *  February 2004
71 > *       If you don't use named rewrite streams, a "default" stream is used.
72 > *
73 > *       Terence Parr, parrt@cs.usfca.edu
74 > *       University of San Francisco
75 > *       February 2004
76   */
77   class TokenStreamRewriteEngine : public TokenStream
78   {
79   public:
80          typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
81 –
82 –        static const size_t MIN_TOKEN_INDEX = 0;
81          static const char* DEFAULT_PROGRAM_NAME;
82 <        static const int PROGRAM_INIT_SIZE = 100;
82 > #ifndef NO_STATIC_CONSTS
83 >        static const size_t MIN_TOKEN_INDEX;
84 >        static const int PROGRAM_INIT_SIZE;
85 > #else
86 >        enum {
87 >                MIN_TOKEN_INDEX = 0,
88 >                PROGRAM_INIT_SIZE = 100
89 >        };
90 > #endif
91  
92          struct tokenToStream {
93                  tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
# Line 102 | Line 108 | class TokenStreamRewriteEngine : public TokenStream (p
108                  {
109                  }
110                  /** Execute the rewrite operation by possibly adding to the buffer.
111 <                 *  Return the index of the next token to operate on.
111 >                 *       Return the index of the next token to operate on.
112                   */
113                  virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
114                          return index;
# Line 191 | Line 197 | class TokenStreamRewriteEngine : public TokenStream (p
197          }
198  
199          /** Rollback the instruction stream for a program so that
200 <         *  the indicated instruction (via instructionIndex) is no
201 <         *  longer in the stream.  UNTESTED!
200 >         *       the indicated instruction (via instructionIndex) is no
201 >         *       longer in the stream.  UNTESTED!
202           */
203          void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
204                                            size_t instructionIndex );
# Line 224 | Line 230 | class TokenStreamRewriteEngine : public TokenStream (p
230          }
231  
232          void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
233 <                                                   size_t index,
233 >                                                        size_t index,
234                                                          const ANTLR_USE_NAMESPACE(std)string& text )
235          {
236                  // to insert after, just insert before next index (even if past end)
# Line 347 | Line 353 | class TokenStreamRewriteEngine : public TokenStream (p
353                                                                    size_t start, size_t end ) const;
354  
355          void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
356 <                return toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
356 >                toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
357          }
358  
359          void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
360                                                  const ANTLR_USE_NAMESPACE(std)string& programName ) const
361          {
362 <                return toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
362 >                toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
363          }
364  
365          void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
366                                                  size_t start, size_t end ) const
367          {
368 <                return toStream(out, DEFAULT_PROGRAM_NAME, start, end);
368 >                toStream(out, DEFAULT_PROGRAM_NAME, start, end);
369          }
370  
371          void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
# Line 367 | Line 373 | class TokenStreamRewriteEngine : public TokenStream (p
373                                                  size_t firstToken, size_t lastToken ) const;
374  
375          void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
376 <                return toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
376 >                toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
377          }
378  
379          void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
# Line 399 | Line 405 | class TokenStreamRewriteEngine : public TokenStream (p
405  
406   protected:
407          /** If op.index > lastRewriteTokenIndexes, just add to the end.
408 <         *  Otherwise, do linear */
408 >         *       Otherwise, do linear */
409          void addToSortedRewriteList(RewriteOperation* op) {
410                  addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
411          }

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines