diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 7907e86..52372cd 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -17,6 +17,7 @@ use self::token::Token; enum State { Initial, Identifier, + Dot, Hash, Comment, String, @@ -90,7 +91,8 @@ impl Lexer { *token = Some(Token::RightParen(c.to_string())); } else if c.is_dot() { - *token = Some(Token::Dot); + self.state = State::Dot; + self.advance(); } else if c.is_hash() { self.state = State::Hash; @@ -141,6 +143,16 @@ impl Lexer { } } + fn state_dot(&mut self, c: char, token: &mut Option) { + if c.is_identifier_delimiter() { + *token = Some(Token::Dot); + self.retract(); + } + else { + assert!(false, "Invalid token character: '{}'", c); + } + } + fn state_hash(&mut self, c: char, token: &mut Option) { if c.is_boolean_true() || c.is_boolean_false() { self.advance(); @@ -195,6 +207,7 @@ impl Iterator for Lexer { match self.state { State::Initial => self.state_initial(c, &mut token), State::Identifier => self.state_identifier(c, &mut token), + State::Dot => self.state_dot(c, &mut token), State::Hash => self.state_hash(c, &mut token), State::String => self.state_string(c, &mut token), State::Comment => self.state_comment(c, &mut token), @@ -229,6 +242,11 @@ mod tests { #[test] fn lexer_finds_dots() { check_single_token(".", Token::Dot); + + let mut lexer = Lexer::new("abc . abc"); + assert_next_token(&mut lexer, &Token::Identifier(String::from("abc"))); + assert_next_token(&mut lexer, &Token::Dot); + assert_next_token(&mut lexer, &Token::Identifier(String::from("abc"))); } #[test]