eryn/sibil
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make the Lex at the end, make tokens in state methods

Browse source
This commit is contained in:
Eryn Wells 2016-12-25 14:20:16 -07:00
parent 0d89cfefc1
commit 7e784ffce1
1 changed files with 31 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

50
src/lexer/mod.rs
View file

@ -78,16 +78,16 @@ impl Lexer {
impl Lexer { impl Lexer {
/// Handle self.state == State::Initial /// Handle self.state == State::Initial
fn state_initial(&mut self, c: char, lex: &mut Option<Lex>) { fn state_initial(&mut self, c: char, token: &mut Option<Token>) {
if c.is_left_paren() { if c.is_left_paren() {
*lex = Some(Lex::new(Token::LeftParen(c.to_string()))); *token = Some(Token::LeftParen(c.to_string()));
} }
else if c.is_right_paren() { else if c.is_right_paren() {
*lex = Some(Lex::new(Token::RightParen(c.to_string()))); *token = Some(Token::RightParen(c.to_string()));
} }
else if c.is_identifier_single() { else if c.is_identifier_single() {
*lex = Some(Lex::new(Token::Identifier(c.to_string()))); *token = Some(Token::Identifier(c.to_string()));
} }
else if c.is_identifier_initial() { else if c.is_identifier_initial() {
self.state = State::Identifier; self.state = State::Identifier;
@ -105,31 +105,36 @@ impl Lexer {
} }
self.advance_begin(); self.advance_begin();
} }
else if c.is_comment_initial() {
self.state = State::Comment;
self.advance();
}
} }
/// Handle self.state == State::Identifier /// Handle self.state == State::Identifier
fn state_identifier(&mut self, c: char, lex: &mut Option<Lex>) { fn state_identifier(&mut self, c: char, token: &mut Option<Token>) {
if c.is_identifier_subsequent() { if c.is_identifier_subsequent() {
// State in Identifier state. // State in Identifier state.
self.advance(); self.advance();
} }
else { else {
*lex = Some(Lex::new(Token::Identifier(self.value()))); *token = Some(Token::Identifier(self.value()));
self.retract(); self.retract();
} }
} }
fn state_hash(&mut self, c: char, lex: &mut Option<Lex>) { fn state_hash(&mut self, c: char, token: &mut Option<Token>) {
if c.is_boolean_true() || c.is_boolean_false() { if c.is_boolean_true() || c.is_boolean_false() {
self.advance(); self.advance();
*lex = Some(Lex::new(Token::Boolean(c.is_boolean_true())); *token = Some(Token::Boolean(c.is_boolean_true()));
} }
} }
fn state_comment(&mut self, c: char, lex: &mut Option<Lex>) { fn state_comment(&mut self, c: char, token: &mut Option<Token>) {
if c.is_newline() { if c.is_newline() {
self.handle_newline(); self.handle_newline();
*lex = Some(Lex::new(Token::Comment(self.value()))); *token = Some(Token::Comment(self.value()));
} }
} }
} }
@ -142,23 +147,26 @@ impl Iterator for Lexer {
if self.begin == self.input.len() { if self.begin == self.input.len() {
return None; return None;
} }
let mut lex: Option<Lex> = None; let mut token: Option<Token> = None;
println!("Lexing '{}'", &self.input[self.begin ..]); println!("Lexing '{}'", &self.input[self.begin ..]);
while lex.is_none() { while token.is_none() {
if let Some(c) = self.input.char_at(self.forward) { let c = match self.input.char_at(self.forward) {
Some(c) => c,
None => '\0',
};
println!("{:?}! c='{}'", self.state, c); println!("{:?}! c='{}'", self.state, c);
match self.state { match self.state {
State::Initial => self.state_initial(c, &mut lex), State::Initial => self.state_initial(c, &mut token),
State::Identifier => self.state_identifier(c, &mut lex), State::Identifier => self.state_identifier(c, &mut token),
State::Hash => self.state_hash(c, &mut lex), State::Hash => self.state_hash(c, &mut token),
} State::Comment => self.state_comment(c, &mut token),
}
else {
assert!(false, "Invalid character! :-(");
} }
} }
self.advance_begin(); self.advance_begin();
assert!(lex.is_some(), "We quit the lexing loop but didn't actually have a token. :-("); let mut lex: Option<Lex> = None;
if let Some(token) = token {
lex = Some(Lex::new(token));
}
lex lex
} }
} }