eryn/sibil
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Convert all the things

Browse source
This commit is contained in:
Eryn Wells 2016-12-27 12:10:15 -07:00
parent c3113742a3
commit 606f2853a9
1 changed files with 66 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

100
src/lexer/mod.rs
View file

@ -20,6 +20,10 @@ use self::token::Token;
type StateResult = Result<Option<Token>, String>; type StateResult = Result<Option<Token>, String>;
trait HasResult {
fn has_token(&self) -> bool;
}
#[derive(Debug)] #[derive(Debug)]
enum State { enum State {
Comment, Comment,
@ -180,8 +184,9 @@ impl Lexer {
self.advance(); self.advance();
} }
else if c.is_identifier_delimiter() { else if c.is_identifier_delimiter() {
self.token_result(Token::Identifier(self.value())); let value = self.value();
self.retract(); self.retract();
return self.token_result(Token::Identifier(value));
} }
else { else {
return self.generic_error(c); return self.generic_error(c);
@ -189,6 +194,7 @@ impl Lexer {
Ok(None) Ok(None)
} }
/// Handle self.state == State::Dot
fn state_dot(&mut self, c: char) -> StateResult { fn state_dot(&mut self, c: char) -> StateResult {
if c.is_identifier_delimiter() { if c.is_identifier_delimiter() {
self.retract(); self.retract();
@ -201,11 +207,12 @@ impl Lexer {
self.advance(); self.advance();
} }
else { else {
self.generic_error(c); return self.generic_error(c);
} }
Ok(None) Ok(None)
} }
/// Handle self.state == State::Hash
fn state_hash(&mut self, c: char) -> StateResult { fn state_hash(&mut self, c: char) -> StateResult {
if c.is_boolean_true() || c.is_boolean_false() { if c.is_boolean_true() || c.is_boolean_false() {
self.advance(); self.advance();
@ -226,12 +233,13 @@ impl Lexer {
self.advance(); self.advance();
} }
else { else {
self.generic_error(c); return self.generic_error(c);
} }
Ok(None) Ok(None)
} }
fn state_number(&mut self, c: char, token: &mut Option<Token>) { /// Handle self.state == State::Number
fn state_number(&mut self, c: char) -> StateResult {
if c.is_digit(self.number_builder.radix_value()) { if c.is_digit(self.number_builder.radix_value()) {
self.number_builder.extend_value(c); self.number_builder.extend_value(c);
self.advance(); self.advance();
@ -241,15 +249,16 @@ impl Lexer {
self.advance(); self.advance();
} }
else if c.is_identifier_delimiter() { else if c.is_identifier_delimiter() {
*token = Some(Token::Number(self.number_builder.resolve()));
self.retract(); self.retract();
return self.token_result(Token::Number(self.number_builder.resolve()));
} }
else { else {
assert!(false, "Invalid token character: '{}'", c); return self.generic_error(c);
} }
Ok(None)
} }
fn state_number_exactness(&mut self, c: char, token: &mut Option<Token>) { fn state_number_exactness(&mut self, c: char) -> StateResult {
if c.is_hash() { if c.is_hash() {
self.state = State::Hash; self.state = State::Hash;
self.advance(); self.advance();
@ -265,25 +274,27 @@ impl Lexer {
self.advance(); self.advance();
} }
else { else {
assert!(false, "Invalid token character: '{}'", c); return self.generic_error(c);
} }
Ok(None)
} }
fn state_number_decimal(&mut self, c: char, token: &mut Option<Token>) { fn state_number_decimal(&mut self, c: char) -> StateResult {
if c.is_digit(Radix::Dec.value()) { if c.is_digit(Radix::Dec.value()) {
self.number_builder.extend_decimal_value(c); self.number_builder.extend_decimal_value(c);
self.advance(); self.advance();
} }
else if c.is_identifier_delimiter() { else if c.is_identifier_delimiter() {
*token = Some(Token::Number(self.number_builder.resolve()));
self.retract(); self.retract();
return self.token_result(Token::Number(self.number_builder.resolve()));
} }
else { else {
assert!(false, "Invalid token character: '{}'", c); return self.generic_error(c);
} }
Ok(None)
} }
fn state_number_radix(&mut self, c: char, token: &mut Option<Token>) { fn state_number_radix(&mut self, c: char) -> StateResult {
if c.is_digit(self.number_builder.radix_value()) { if c.is_digit(self.number_builder.radix_value()) {
self.number_builder.extend_value(c); self.number_builder.extend_value(c);
self.state = State::Number; self.state = State::Number;
@ -303,11 +314,12 @@ impl Lexer {
self.advance(); self.advance();
} }
else { else {
assert!(false, "Invalid token character: '{}'", c); return self.generic_error(c);
} }
Ok(None)
} }
fn state_number_sign(&mut self, c: char, token: &mut Option<Token>) { fn state_number_sign(&mut self, c: char) -> StateResult {
if c.is_digit(self.number_builder.radix_value()) { if c.is_digit(self.number_builder.radix_value()) {
self.number_builder.extend_value(c); self.number_builder.extend_value(c);
self.state = State::Number; self.state = State::Number;
@ -318,42 +330,46 @@ impl Lexer {
self.advance(); self.advance();
} }
else { else {
assert!(false, "Invalid token character: '{}'", c); return self.generic_error(c);
} }
Ok(None)
} }
fn state_sign(&mut self, c: char, token: &mut Option<Token>) { fn state_sign(&mut self, c: char) -> StateResult {
if c.is_digit(Radix::Dec.value()) { if c.is_digit(Radix::Dec.value()) {
self.number_builder.extend_value(c); self.number_builder.extend_value(c);
self.state = State::Number; self.state = State::Number;
self.advance(); self.advance();
} }
else if c.is_identifier_delimiter() { else if c.is_identifier_delimiter() {
*token = Some(Token::Identifier(self.value())); let value = self.value();
self.retract(); self.retract();
return self.token_result(Token::Identifier(value));
} }
else { else {
assert!(false, "Invalid token character: '{}'", c); return self.generic_error(c);
} }
Ok(None)
} }
fn state_string(&mut self, c: char, token: &mut Option<Token>) { fn state_string(&mut self, c: char) -> StateResult {
self.advance(); self.advance();
if c.is_string_quote() { if c.is_string_quote() {
*token = Some(Token::String(self.value())); return self.token_result(Token::String(self.value()));
} }
Ok(None)
} }
fn state_comment(&mut self, c: char, token: &mut Option<Token>) { fn state_comment(&mut self, c: char) -> StateResult {
if c.is_newline() { if c.is_newline() {
self.handle_newline(); self.handle_newline();
*token = Some(Token::Comment(self.value())); return self.token_result(Token::Comment(self.value()));
} }
else if c.is_eof() { else if c.is_eof() {
*token = Some(Token::Comment(self.value())); return self.token_result(Token::Comment(self.value()));
} }
// Consume all characters.
self.advance(); self.advance();
Ok(None)
} }
} }
@ -374,21 +390,25 @@ impl Iterator for Lexer {
}; };
println!("{:?}! c='{}'", self.state, c); println!("{:?}! c='{}'", self.state, c);
let previous_forward = self.forward; let previous_forward = self.forward;
match self.state { let result = match self.state {
State::Initial => self.state_initial(c), State::Initial => self.state_initial(c),
State::Identifier => self.state_identifier(c), State::Identifier => self.state_identifier(c),
State::Dot => self.state_dot(c), State::Dot => self.state_dot(c),
State::Hash => self.state_hash(c), State::Hash => self.state_hash(c),
State::Number => self.state_number(c, &mut token), State::Number => self.state_number(c),
State::NumberExactness => self.state_number_exactness(c, &mut token), State::NumberExactness => self.state_number_exactness(c),
State::NumberDecimal => self.state_number_decimal(c, &mut token), State::NumberDecimal => self.state_number_decimal(c),
State::NumberRadix => self.state_number_radix(c, &mut token), State::NumberRadix => self.state_number_radix(c),
State::NumberSign => self.state_number_sign(c, &mut token), State::NumberSign => self.state_number_sign(c),
State::Sign => self.state_sign(c, &mut token), State::Sign => self.state_sign(c),
State::String => self.state_string(c, &mut token), State::String => self.state_string(c),
State::Comment => self.state_comment(c, &mut token), State::Comment => self.state_comment(c),
};
assert!(result.has_token() || self.forward != previous_forward, "No lexing progress made!");
if result.has_token() {
token = result.ok().unwrap();
break;
} }
assert!(token.is_some() || self.forward != previous_forward, "No lexing progress made!");
} }
self.advance_begin(); self.advance_begin();
match token { match token {
@ -398,6 +418,18 @@ impl Iterator for Lexer {
} }
} }
impl HasResult for StateResult {
fn has_token(&self) -> bool {
match *self {
Ok(ref token) => match *token {
Some(_) => true,
None => false,
},
Err(_) => false
}
}
}
// //
// UNIT TESTING // UNIT TESTING
// //