Character checks for identifier initial and subsequent

lexer/src/chars.rs

@@ -5,9 +5,49 @@
 pub trait Lexable {
     fn is_left_paren(&self) -> bool;
     fn is_right_paren(&self) -> bool;
+    fn is_identifier_initial(&self) -> bool;
+    fn is_identifier_subsequent(&self) -> bool;
+    fn is_identifier_delimiter(&self) -> bool;
 }
 
 impl Lexable for char {
-    fn is_left_paren(&self) -> bool { *self == '(' }
-    fn is_right_paren(&self) -> bool { *self == ')' }
+    fn is_left_paren(&self) -> bool {
+        *self == '('
+    }
+
+    fn is_right_paren(&self) -> bool {
+        *self == ')'
+    }
+
+    fn is_identifier_initial(&self) -> bool {
+        self.is_alphabetic() || self.is_special_initial()
+    }
+
+    fn is_identifier_subsequent(&self) -> bool {
+        self.is_identifier_initial() || self.is_numeric() || self.is_special_subsequent()
+    }
+
+    fn is_identifier_delimiter(&self) -> bool {
+        self.is_whitespace() || self.is_left_paren() || self.is_right_paren()
+    }
+}
+
+trait LexableSpecial {
+    fn is_special_initial(&self) -> bool;
+    fn is_special_subsequent(&self) -> bool;
+    fn is_explicit_sign(&self) -> bool;
+}
+
+impl LexableSpecial for char {
+    fn is_special_initial(&self) -> bool {
+        "!$%&*/:<=>?~_^".contains(*self)
+    }
+
+    fn is_special_subsequent(&self) -> bool {
+        self.is_explicit_sign() || ".@".contains(*self)
+    }
+
+    fn is_explicit_sign(&self) -> bool {
+        *self == '+' || *self == '-'
+    }
 }

lexer/src/lib.rs

@@ -55,7 +55,7 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
                     Some(c) if c.is_left_paren() => self.emit(Token::LeftParen, Resume::AtNext),
                     Some(c) if c.is_right_paren() => self.emit(Token::RightParen, Resume::AtNext),
                     Some(c) if c.is_whitespace() => IterationResult::Continue,
-                    Some(c) if c.is_alphabetic() => {
+                    Some(c) if c.is_identifier_initial() => {
                         buffer.push(c);
                         IterationResult::Continue
                     },
@@ -66,11 +66,11 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
             }
             else {
                 match peek {
-                    Some(c) if c.is_alphabetic() => {
+                    Some(c) if c.is_identifier_subsequent() => {
                         buffer.push(c);
                         IterationResult::Continue
                     }
-                    Some(c) if c.is_left_paren() || c.is_right_paren() || c.is_whitespace() =>
+                    Some(c) if c.is_identifier_delimiter() =>
                         self.emit(Token::Id(buffer.clone()), Resume::Here),
                     Some(c) => self.fail(format!("Invalid character: {}", c)),
                     // Found EOF. Emit what we have and finish.