diff --git a/src/lexer.rs b/src/lexer.rs index ac7a7da..2ca4ce5 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -85,17 +85,21 @@ impl Lexer { println!("< forward={}", self.forward); } + /// Advance the begin pointer to prepare for the next iteration. fn advance_begin(&mut self) { self.begin = self.input.index_after(self.forward); println!("> begin={}", self.begin); } + /// Get the substring between the two input indexes. This is the value to give to a new Token + /// instance. fn value(&self) -> String { self.input[self.begin .. self.forward].to_string() } } impl Lexer { + /// Handle self.state == State::Initial fn state_initial(&mut self, c: char, token: &mut Option) { println!("Initial! c='{}'", c); if c.is_left_paren() { @@ -110,6 +114,7 @@ impl Lexer { } } + /// Handle self.state == State::Identifier fn state_identifier(&mut self, c: char, token: &mut Option) { if c.is_identifier_subsequent() { // State in Identifier state. @@ -132,7 +137,7 @@ impl Iterator for Lexer { } let mut token: Option = None; println!("Lexing '{}'", &self.input[self.begin ..]); - loop { + while token.is_none() { if let Some(c) = self.input.char_at(self.forward) { match self.state { State::Initial => self.state_initial(c, &mut token), @@ -142,12 +147,9 @@ impl Iterator for Lexer { else { assert!(false, "Invalid character! :-("); } - if token.is_some() { - break; - } } self.advance_begin(); - assert!(token.is_some()); + assert!(token.is_some(), "We quit the lexing loop but didn't actually have a token. :-("); token } }