Add Comment token

src/lexer/char.rs

@@ -14,6 +14,7 @@ pub trait Lexable {
     fn is_boolean_true(&self) -> bool;
     fn is_boolean_false(&self) -> bool;
     fn is_newline(&self) -> bool;
+    fn is_comment_initial(&self) -> bool;
 }
 
 impl Lexable for char {
@@ -52,5 +53,9 @@ impl Lexable for char {
     fn is_newline(&self) -> bool {
         *self == '\n'
     }
+
+    fn is_comment_initial(&self) -> bool {
+        *self == ';'
+    }
 }
 

src/lexer/mod.rs

@@ -18,6 +18,7 @@ enum State {
     Initial,
     Identifier,
     Hash,
+    Comment,
 }
 
 pub struct Lexer {
@@ -65,6 +66,10 @@ impl Lexer {
         println!("> begin={}, forward={}", self.begin, self.forward);
     }
 
+    fn handle_newline(&mut self) {
+        self.line += 1;
+    }
+
     /// Get the substring between the two input indexes. This is the value to give to a new Token instance.
     fn value(&self) -> String {
         self.input[self.begin .. self.forward].to_string()
@@ -96,7 +101,7 @@ impl Lexer {
 
         else if c.is_whitespace() {
             if c.is_newline() {
-                self.line += 1;
+                self.handle_newline();
             }
             self.advance_begin();
         }
@@ -120,6 +125,13 @@ impl Lexer {
             *lex = Some(Lex::new(Token::Boolean(c.is_boolean_true()));
         }
     }
+
+    fn state_comment(&mut self, c: char, lex: &mut Option<Lex>) {
+        if c.is_newline() {
+            self.handle_newline();
+            *lex = Some(Lex::new(Token::Comment(self.value())));
+        }
+    }
 }
 
 impl Iterator for Lexer {

src/lexer/token.rs

@@ -8,6 +8,7 @@ pub enum Token {
     RightParen(String),
     Identifier(String),
     Boolean(bool),
+    Comment(String),
 }
 
 /// A Lex is a Token extracted from a specific position in an input. It contains useful information about the token's