diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index b3d66ef..f7ad002 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -12,6 +12,7 @@ enum Resume { Here, AtNext } #[derive(Debug, Eq, PartialEq)] enum IterationResult { + Finish, Continue, Emit(Token, Resume), Error(String), @@ -40,31 +41,39 @@ impl Iterator for Lexer where T: Iterator { fn next(&mut self) -> Option { let mut buffer = String::new(); - while let Some(peek) = self.input.peek().map(char::clone) { + loop { + let peek = self.input.peek().map(char::clone); let result = if buffer.is_empty() { match peek { - '(' => self.emit(Token::LeftParen, Resume::AtNext), - ')' => self.emit(Token::RightParen, Resume::AtNext), - c if c.is_whitespace() => IterationResult::Continue, - c if c.is_alphabetic() => { + Some('(') => self.emit(Token::LeftParen, Resume::AtNext), + Some(')') => self.emit(Token::RightParen, Resume::AtNext), + Some(c) if c.is_whitespace() => IterationResult::Continue, + Some(c) if c.is_alphabetic() => { buffer.push(c); IterationResult::Continue }, - c => self.fail(format!("Invalid character: {}", c)), + Some(c) => self.fail(format!("Invalid character: {}", c)), + // We found EOF and there's no pending string, so just finish. + None => IterationResult::Finish, } } else { match peek { - c if c.is_alphabetic() => { + Some(c) if c.is_alphabetic() => { buffer.push(c); IterationResult::Continue } - c if c == '(' || c == ')' || c.is_whitespace() => + Some(c) if c == '(' || c == ')' || c.is_whitespace() => self.emit(Token::Id(buffer.clone()), Resume::Here), - c => self.fail(format!("Invalid character: {}", c)), + Some(c) => self.fail(format!("Invalid character: {}", c)), + // Found EOF. Emit what we have and finish. + // Note: the Resume argument doesn't matter in this case since the input + // iterator will always be None from here on. + None => self.emit(Token::Id(buffer.clone()), Resume::Here), } }; match result { + IterationResult::Finish => break, IterationResult::Continue => self.input.next(), IterationResult::Emit(token, resume) => { if resume == Resume::AtNext {