eryn/sibil
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Check for EOF (aka input.next() -> None) and emit what we have before finishing forever

Browse source
This commit is contained in:
Eryn Wells 2017-05-02 21:44:57 -07:00
parent 5fe10fe002
commit 28e5814101
1 changed files with 18 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

27
lexer/src/lib.rs
View file

@ -12,6 +12,7 @@ enum Resume { Here, AtNext }
#[derive(Debug, Eq, PartialEq)]
enum IterationResult {
Finish,
Continue,
Emit(Token, Resume),
Error(String),
@ -40,31 +41,39 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
fn next(&mut self) -> Option<Self::Item> {
let mut buffer = String::new();
while let Some(peek) = self.input.peek().map(char::clone) {
loop {
let peek = self.input.peek().map(char::clone);
let result = if buffer.is_empty() {
match peek {
'(' => self.emit(Token::LeftParen, Resume::AtNext),
')' => self.emit(Token::RightParen, Resume::AtNext),
c if c.is_whitespace() => IterationResult::Continue,
c if c.is_alphabetic() => {
Some('(') => self.emit(Token::LeftParen, Resume::AtNext),
Some(')') => self.emit(Token::RightParen, Resume::AtNext),
Some(c) if c.is_whitespace() => IterationResult::Continue,
Some(c) if c.is_alphabetic() => {
buffer.push(c);
IterationResult::Continue
},
c => self.fail(format!("Invalid character: {}", c)),
Some(c) => self.fail(format!("Invalid character: {}", c)),
// We found EOF and there's no pending string, so just finish.
None => IterationResult::Finish,
}
}
else {
match peek {
c if c.is_alphabetic() => {
Some(c) if c.is_alphabetic() => {
buffer.push(c);
IterationResult::Continue
}
c if c == '(' || c == ')' || c.is_whitespace() =>
Some(c) if c == '(' || c == ')' || c.is_whitespace() =>
self.emit(Token::Id(buffer.clone()), Resume::Here),
c => self.fail(format!("Invalid character: {}", c)),
Some(c) => self.fail(format!("Invalid character: {}", c)),
// Found EOF. Emit what we have and finish.
// Note: the Resume argument doesn't matter in this case since the input
// iterator will always be None from here on.
None => self.emit(Token::Id(buffer.clone()), Resume::Here),
}
};
match result {
IterationResult::Finish => break,
IterationResult::Continue => self.input.next(),
IterationResult::Emit(token, resume) => {
if resume == Resume::AtNext {