eryn/sibil
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use a string to build up the value of String tokens, so we can properly handle escape characters and such

Browse source
This commit is contained in:
Eryn Wells 2016-12-28 18:01:36 -05:00
parent 872d07e6b7
commit c881ee874a
1 changed files with 23 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/lexer/mod.rs
View file

@ -57,6 +57,7 @@ pub struct Lexer {
line_offset: usize, line_offset: usize,
state: State, state: State,
number_builder: NumberBuilder, number_builder: NumberBuilder,
string_value: String,
} }
impl Lexer { impl Lexer {
@ -69,6 +70,7 @@ impl Lexer {
line_offset: 1, line_offset: 1,
state: State::Initial, state: State::Initial,
number_builder: NumberBuilder::new(), number_builder: NumberBuilder::new(),
string_value: String::new(),
} }
} }
} }
@ -145,6 +147,7 @@ impl Lexer {
return self.token_result(Token::Quote); return self.token_result(Token::Quote);
} }
else if c.is_string_quote() { else if c.is_string_quote() {
self.string_value = String::from("");
self.state = State::String; self.state = State::String;
self.advance(); self.advance();
} }
@ -431,22 +434,29 @@ impl Lexer {
fn state_string(&mut self, c: char) -> StateResult { fn state_string(&mut self, c: char) -> StateResult {
self.advance(); self.advance();
if c.is_string_quote() { if c.is_string_quote() {
return self.token_result(Token::String(self.value())); return self.token_result(Token::String(self.string_value.clone()));
} }
else if c.is_string_escape_leader() { else if c.is_string_escape_leader() {
self.state = State::StringEscape; self.state = State::StringEscape;
} }
else {
self.string_value.push(c);
}
Ok(None) Ok(None)
} }
fn state_string_escape(&mut self, c: char) -> StateResult { fn state_string_escape(&mut self, c: char) -> StateResult {
if c.is_string_escaped() { let char_to_push = match c {
'0' => '\0',
'n' => '\n',
't' => '\t',
'"' => '"',
'\\' => '\\',
_ => return Err(self.error_string(format!("Invalid string escape character: {}", c))),
};
self.string_value.push(char_to_push);
self.state = State::String; self.state = State::String;
self.advance(); self.advance();
}
else {
return Err(self.error_string(format!("Invalid string escape character: {}", c)));
}
Ok(None) Ok(None)
} }
@ -652,8 +662,9 @@ mod tests {
#[test] #[test]
fn finds_escaped_characters_in_strings() { fn finds_escaped_characters_in_strings() {
check_single_token("\"\\\\\"", Token::String(String::from("\"\\\\\""))); check_single_token("\"\\\\\"", Token::String(String::from("\\")));
check_single_token("\"\\\"\"", Token::String(String::from("\"\\\"\""))); check_single_token("\"\\\"\"", Token::String(String::from("\"")));
check_single_token("\"\\n\"", Token::String(String::from("\n")));
} }
#[test] #[test]
@ -705,13 +716,12 @@ mod tests {
#[test] #[test]
fn finds_strings() { fn finds_strings() {
check_single_token("\"\"", Token::String(String::from("\"\""))); check_single_token("\"\"", Token::String(String::from("")));
check_single_token("\"abc\"", Token::String(String::from("\"abc\""))); check_single_token("\"abc\"", Token::String(String::from("abc")));
} }
#[test] #[test]
fn lexes_simple_expression() { fn lexes_simple_expression() {
let mut lexer = Lexer::new("(+ 3.4 6.8)");
check_tokens("(+ 3.4 6.8)", vec![ check_tokens("(+ 3.4 6.8)", vec![
Token::LeftParen(String::from("(")), Token::LeftParen(String::from("(")),
Token::Identifier(String::from("+")), Token::Identifier(String::from("+")),