Move the lexer to its own sibillexer module

Lots of failing tests right now, unfortunately. :(
This commit is contained in:
Eryn Wells 2017-04-15 09:37:12 -07:00
parent cea63e8e8e
commit fc947280ae
9 changed files with 264 additions and 244 deletions

View file

@ -1,35 +0,0 @@
/* token.rs
* Eryn Wells <eryn@erynwells.me>
*/
use types::{Bool, Char, Number};
#[derive(Debug, PartialEq)]
pub enum Token {
Boolean(Bool),
Character(Char),
Comment(String),
Dot,
Id(String),
LeftParen,
LeftVectorParen,
Number(Number),
Quote,
RightParen,
String(String),
}
/// A Lex is a Token extracted from a specific position in an input. It contains useful information about the token's
/// place.
#[derive(Debug)]
pub struct Lex {
pub token: Token,
pub line: usize,
pub offset: usize,
}
impl Lex {
pub fn new(token: Token, line: usize, offset: usize) -> Lex {
Lex { token: token, line: line, offset: offset }
}
}