View Javadoc

1   /*
2    * LinkedListTokenStream.java
3    * 
4    * Copyright (c) 2006 David Holroyd
5    *
6    * Licensed under the Apache License, Version 2.0 (the "License");
7    * you may not use this file except in compliance with the License.
8    * You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package uk.co.badgersinfoil.metaas.impl.antlr;
20  
21  import org.antlr.runtime.CharStream;
22  import org.antlr.runtime.Token;
23  import org.antlr.runtime.TokenSource;
24  import org.antlr.runtime.TokenStream;
25  
26  public class LinkedListTokenStream implements TokenStream  {
27  	
28  	protected TokenSource tokenSource;
29  	protected LinkedListToken head;
30  	protected LinkedListToken tail;
31  
32  	/** Skip tokens on any channel but this one; this is how we skip whitespace... */
33  	protected int channel = Token.DEFAULT_CHANNEL;
34  
35  	/** By default, track all incoming tokens */
36  	protected boolean discardOffChannelTokens = false;
37  
38  	/** Track the last mark() call result value for use in rewind(). */
39  	protected LinkedListToken lastMarker;
40  
41  	/** The current element in the tokens list (next token
42  	 *  to consume).  p==null indicates that the tokens list is empty
43  	 */
44  	protected LinkedListToken p = null;
45  
46  
47  	public LinkedListTokenStream() {
48  	}
49  
50  	public TokenSource getSource() {
51  		return tokenSource;
52  	}
53  
54  	/**
55  	 * Reverses the stream 'count' tokens back, causing the tokens to be
56  	 * removed from the stream.  Can be used to erase tokens which parser
57  	 * lookahead has summoned, but which represent input to be handled by
58  	 * an 'island grammar'.
59  	 */
60  	public void scrub(int count) {
61  		if (p == null) {
62  			p = tail;
63  		}
64  		for (; count > 0 ; count--) {
65  			p = p.getPrev();
66  		}
67  		p.setNext(null);
68  		tail = p;
69  		p = null;
70  	}
71  
72  	/**
73  	 * The given TokenSource must produce tokens of type LinkedListToken
74  	 */
75  	public LinkedListTokenStream(TokenSource tokenSource) {
76  		this.tokenSource = tokenSource;
77  	}
78  	public LinkedListTokenStream(TokenSource tokenSource, int channel) {
79  		this(tokenSource);
80  		this.channel = channel;
81  	}
82  
83  	/** Reset this token stream by setting its token source. */
84  	public void setTokenSource(TokenSource tokenSource) {
85  		this.tokenSource = tokenSource;
86  		p = null;
87  		channel = Token.DEFAULT_CHANNEL;
88  	}
89  
90  	private LinkedListToken readNextToken() {
91  		LinkedListToken t = (LinkedListToken)tokenSource.nextToken();
92  		while ( t!=null && t.getType()!=CharStream.EOF ) {
93  			boolean discard = false;
94  			if ( discardOffChannelTokens && t.getChannel()!=this.channel ) {
95  				discard = true;
96  			}
97  			if ( !discard )	{
98  				if (head == null && tail == null) {
99  					head = tail = t;
100 				} else {
101 					tail.setNext(t);
102 					t.setPrev(tail);
103 					tail = t;
104 				}
105 				break;
106 			}
107 			t = (LinkedListToken)tokenSource.nextToken();
108 		}
109 		if (t.getType() == CharStream.EOF) {
110 			// prevent ourselves from producing lots of EOF tokens
111 			// if the parser is 'pushy'; also, do the head/tail dance
112 			if (tail!=null && tail.getType()==CharStream.EOF) {
113 				return tail;
114 			} else {
115 				if (head == null && tail == null) {
116 					head = tail = t;
117 				} else {
118 					tail.setNext(t);
119 					t.setPrev(tail);
120 					tail = t;
121 				}
122 			}
123 		}
124 		return skipOffTokenChannels(t);
125 	}
126 	
127 	/**
128 	 * Returns the token that follows the given token in the stream, or
129 	 * null if there's no token following
130 	 */
131 	private LinkedListToken succ(LinkedListToken tok) {
132 		LinkedListToken next = tok.getNext();
133 		if (next == null) {
134 			next = readNextToken();
135 		}
136 		return next;
137 	}
138 
139 
140 	/** Return absolute token i; ignore which channel the tokens are on;
141 	 *  that is, count all tokens not just on-channel tokens.
142 	 */
143 	public Token get(int i) {
144 		LinkedListToken tok = head;
145 		for (int c=0; c<i; c++) {
146 			tok = succ(tok);
147 		}
148 		return tok;
149 	}
150 
151 	public TokenSource getTokenSource() {
152 		return tokenSource;
153 	}
154 
155 	public Token LT(int k) {
156 		if (p == null) {
157 			p = readNextToken();
158 		}
159 		if (k == 0) {
160 			return null;
161 		}
162 		if (k < 0) {
163 			return LB(-k);
164 		}
165 		LinkedListToken i = p;
166 		int n = 1;
167 		// find k good tokens
168 		while (n < k) {
169 			LinkedListToken next = succ(i);
170 			if (i == null) {
171 				return Token.EOF_TOKEN;
172 			}
173 			// skip off-channel tokens
174 			i = skipOffTokenChannels(next); // leave p on valid token
175 			n++;
176 		}
177 		if (i == null) {
178 			return Token.EOF_TOKEN;
179 		}
180 		return i;
181 	}
182 
183 	/** Look backwards k tokens on-channel tokens */
184 	protected Token LB(int k) {
185 		if (p == null) {
186 			p = readNextToken();
187 		}
188 		if (k == 0) {
189 			return null;
190 		}
191 
192 		LinkedListToken i = p;
193 		int n = 1;
194 		// find k good tokens looking backwards
195 		while (n <= k) {
196 			LinkedListToken next = i.getPrev();
197 			if (next == null) {
198 				return null;
199 			}
200 			// skip off-channel tokens
201 			i = skipOffTokenChannelsReverse(next); // leave p on valid token
202 			n++;
203 		}
204 		return i;
205 	}
206 
207 
208 	public String toString(int start, int stop) {
209 		LinkedListToken tok = head;
210 		int i = 0;
211 		for (; i<start && tok!=null; i++) {
212 			tok = succ(tok);
213 		}
214  		StringBuffer buf = new StringBuffer();
215 		for (; i<=stop && tok!=null; i++) {
216 			buf.append(tok.getText());
217 			tok = succ(tok);
218 		}
219  		return buf.toString();
220 	}
221 
222 	public String toString(Token start, Token stop) {
223 		LinkedListToken tok = (LinkedListToken)start;
224  		StringBuffer buf = new StringBuffer();
225 		do {
226 			buf.append(tok.getText());
227 			tok = succ(tok);
228 		} while (tok!=null && tok!=stop);
229  		return buf.toString();
230 	}
231 
232 	public void consume() {
233 		do {
234 			p = p.getNext();
235 		} while (p != null && p.getChannel() != channel);
236 	}
237 
238 	public int index() {
239 		int i=0;
240 		for (LinkedListToken tok=head; tok!=p&&tok!=null; tok=tok.getNext()) {
241 			i++;
242 		}
243 		return i;
244 	}
245 
246 	public int LA(int i) {
247 	        return LT(i).getType();
248 	}
249 
250 	public int mark() {
251 		// TODO: could store marks in a hash; does it make any difference?
252 		lastMarker = (LinkedListToken)LT(1);
253 		return index();
254 	}
255 
256 	public void release(int marker) {
257 		// no resources to release
258 	}
259 
260 	public void rewind() {
261 		p = lastMarker;
262 	}
263 
264 	public void rewind(int marker) {
265 		seek(marker);
266 	}
267 
268 	public void seek(int index) {
269 		p = head;
270 		for (int i=0; i<index; i++) {
271 			p = succ(p);
272 		}
273 	}
274 
275 	public int size() {
276 		int s = 0;
277 		for (LinkedListToken tok=head; tok!=null; tok=tok.getNext()) {
278 			s++;
279 		}
280 		return s;
281 	}
282 
283 
284 	public void discardOffChannelTokens(boolean discardOffChannelTokens) {
285 		this.discardOffChannelTokens = discardOffChannelTokens;
286 	}
287 
288 	/**
289 	 * Given a starting token, return the first on-channel token.
290 	 */
291 	protected LinkedListToken skipOffTokenChannels(LinkedListToken i) {
292 		while (i != null && i.getChannel() != channel) {
293 			i = succ(i);
294 		}
295 		return i;
296 	}
297 
298 	protected LinkedListToken skipOffTokenChannelsReverse(LinkedListToken i) {
299 		while (i != null && i.getChannel() != channel) {
300 			i = i.getPrev();
301 		}
302 		return i;
303 	}
304 }