diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index 393a6de..2220eb6 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -49,16 +49,14 @@ impl Lexer where T: Iterator { fn prepare_offsets(&mut self) { } fn update_offsets(&mut self, c: char) { - self.offset += 1; match c { '\n' => { self.line += 1; self.offset = 0; }, - _ => { - self.offset += 1; - }, + _ => self.offset += 1 } + println!("incremented offsets {}:{}", self.line, self.offset); } } @@ -68,6 +66,10 @@ impl Iterator for Lexer where T: Iterator { fn next(&mut self) -> Option { self.prepare_offsets(); + let mut token_line = self.line; + let mut token_offset = self.offset; + println!("beginning token at {}:{}", token_line, token_offset); + let mut buffer = String::new(); let mut state: Box = Box::new(Begin::new()); let mut out: Option = None; @@ -78,7 +80,7 @@ impl Iterator for Lexer where T: Iterator { None => match state.none() { Ok(None) => break, Ok(Some(token)) => { - out = Some(Ok(Lex::new(token, &buffer, self.line, self.offset))); + out = Some(Ok(Lex::new(token, &buffer, token_line, token_offset))); break; }, Err(err) => self.handle_error(err) @@ -101,13 +103,15 @@ impl Iterator for Lexer where T: Iterator { if resume == Resume::AtNext { self.next(); } + token_line = self.line; + token_offset = self.offset; }, StateResult::Emit(token, resume) => { if resume == Resume::AtNext { buffer.push(c); self.next(); } - out = Some(Ok(Lex::new(token, &buffer, self.line, self.offset))); + out = Some(Ok(Lex::new(token, &buffer, token_line, token_offset))); break; }, StateResult::Fail(err) => self.handle_error(err), @@ -115,6 +119,7 @@ impl Iterator for Lexer where T: Iterator { }, } } + println!("emitting {:?}", out); out } } diff --git a/lexer/src/token.rs b/lexer/src/token.rs index 0453169..2cc77bb 100644 --- a/lexer/src/token.rs +++ b/lexer/src/token.rs @@ -2,7 +2,7 @@ * Eryn Wells */ -#[derive(Debug, Eq)] +#[derive(Debug, Eq, PartialEq)] pub struct Lex { token: Token, value: String, @@ -33,8 +33,3 @@ impl Lex { pub fn value(&self) -> &str { self.value.as_str() } } -impl PartialEq for Lex { - fn eq(&self, rhs: &Lex) -> bool { - self.token == rhs.token && self.value == rhs.value - } -} diff --git a/lexer/tests/single_tokens.rs b/lexer/tests/single_tokens.rs index bb747f8..0578934 100644 --- a/lexer/tests/single_tokens.rs +++ b/lexer/tests/single_tokens.rs @@ -67,16 +67,24 @@ fn bool_long_false() { #[test] fn bool_with_spaces() { // See issue #12 - let expected_lex = Lex::new(Token::Bool(false), "#f", 0, 0); + let expected_lex = Lex::new(Token::Bool(false), "#f", 0, 2); let mut lex = Lexer::new(" #f ".chars()); assert_eq!(lex.next(), Some(Ok(expected_lex))); assert_eq!(lex.next(), None); } #[test] -fn integer() { +fn simple_integers() { let mut lex = Lexer::new("23 42".chars()); assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(23), "23", 0, 0)))); - assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(42), "42", 0, 0)))); + assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(42), "42", 0, 3)))); + assert_eq!(lex.next(), None); +} + +#[test] +fn integers_in_alternative_bases() { + let mut lex = Lexer::new("#x2A #b11001".chars()); + assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(0x2A), "#x2A", 0, 0)))); + assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(0b11001), "#b11001", 0, 5)))); assert_eq!(lex.next(), None); }