Straighten out test failures

src/lexer/char.rs

@@ -12,11 +12,15 @@ pub trait Lexable {
     fn is_string_quote(&self) -> bool;
     fn is_newline(&self) -> bool;
     fn is_eof(&self) -> bool;
+
     fn is_identifier_initial(&self) -> bool;
     fn is_identifier_subsequent(&self) -> bool;
     fn is_identifier_single(&self) -> bool;
+    fn is_identifier_delimiter(&self) -> bool;
+
     fn is_boolean_true(&self) -> bool;
     fn is_boolean_false(&self) -> bool;
+
     fn is_comment_initial(&self) -> bool;
 }
 
@@ -36,7 +40,7 @@ impl Lexable for char {
     fn is_dot(&self) -> bool {
         *self == '.'
     }
-    
+
     fn is_string_quote(&self) -> bool {
         *self == '"'
     }
@@ -72,4 +76,13 @@ impl Lexable for char {
     fn is_identifier_single(&self) -> bool {
         charset::identifier_singles().contains(&self)
     }
+
+    fn is_identifier_delimiter(&self) -> bool {
+        self.is_whitespace()
+            || self.is_comment_initial()
+            || self.is_left_paren()
+            || self.is_right_paren()
+            || self.is_string_quote()
+            || self.is_eof()
+    }
 }

src/lexer/mod.rs

@@ -132,12 +132,12 @@ impl Lexer {
             // State in Identifier state.
             self.advance();
         }
-        else {
+        else if c.is_identifier_delimiter() {
             *token = Some(Token::Identifier(self.value()));
             self.retract();
         }
         else {
-            assert!(false, "Invalid token character: {}", c);
+            assert!(false, "Invalid token character: '{}'", c);
         }
     }
 
@@ -151,7 +151,7 @@ impl Lexer {
             *token = Some(Token::LeftVectorParen);
         }
         else {
-            assert!(false, "Invalid token character: {}", c);
+            assert!(false, "Invalid token character: '{}'", c);
         }
     }
 
@@ -160,24 +160,18 @@ impl Lexer {
         if c.is_string_quote() {
             *token = Some(Token::String(self.value()));
         }
-        else {
-            assert!(false, "Invalid token character: {}", c);
-        }
     }
 
     fn state_comment(&mut self, c: char, token: &mut Option<Token>) {
         if c.is_newline() {
             self.handle_newline();
-            self.advance();
             *token = Some(Token::Comment(self.value()));
         }
         else if c.is_eof() {
             *token = Some(Token::Comment(self.value()));
-            self.advance();
-        }
-        else {
-            assert!(false, "Invalid token character: {}", c);
         }
+        // Consume all characters.
+        self.advance();
     }
 }