[lexer] Correctly set line/offset for emitted Lexes

lexer/src/lib.rs

@@ -49,16 +49,14 @@ impl<T> Lexer<T> where T: Iterator<Item=char> {
     fn prepare_offsets(&mut self) { }
 
     fn update_offsets(&mut self, c: char) {
-        self.offset += 1;
         match c {
             '\n' => {
                 self.line += 1;
                 self.offset = 0;
             },
-            _ => {
-                self.offset += 1;
-            },
+            _ => self.offset += 1
         }
+        println!("incremented offsets {}:{}", self.line, self.offset);
     }
 }
 
@@ -68,6 +66,10 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
     fn next(&mut self) -> Option<Self::Item> {
         self.prepare_offsets();
 
+        let mut token_line = self.line;
+        let mut token_offset = self.offset;
+        println!("beginning token at {}:{}", token_line, token_offset);
+
         let mut buffer = String::new();
         let mut state: Box<states::State> = Box::new(Begin::new());
         let mut out: Option<Self::Item> = None;
@@ -78,7 +80,7 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
                 None => match state.none() {
                     Ok(None) => break,
                     Ok(Some(token)) => {
-                        out = Some(Ok(Lex::new(token, &buffer, self.line, self.offset)));
+                        out = Some(Ok(Lex::new(token, &buffer, token_line, token_offset)));
                         break;
                     },
                     Err(err) => self.handle_error(err)
@@ -101,13 +103,15 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
                             if resume == Resume::AtNext {
                                 self.next();
                             }
+                            token_line = self.line;
+                            token_offset = self.offset;
                         },
                         StateResult::Emit(token, resume) => {
                             if resume == Resume::AtNext {
                                 buffer.push(c);
                                 self.next();
                             }
-                            out = Some(Ok(Lex::new(token, &buffer, self.line, self.offset)));
+                            out = Some(Ok(Lex::new(token, &buffer, token_line, token_offset)));
                             break;
                         },
                         StateResult::Fail(err) => self.handle_error(err),
@@ -115,6 +119,7 @@ impl<T> Iterator for Lexer<T> where T: Iterator<Item=char> {
                 },
             }
         }
+        println!("emitting {:?}", out);
         out
     }
 }

lexer/src/token.rs

@@ -2,7 +2,7 @@
  * Eryn Wells <eryn@erynwells.me>
  */
 
-#[derive(Debug, Eq)]
+#[derive(Debug, Eq, PartialEq)]
 pub struct Lex {
     token: Token,
     value: String,
@@ -33,8 +33,3 @@ impl Lex {
     pub fn value(&self) -> &str { self.value.as_str() }
 }
 
-impl PartialEq for Lex {
-    fn eq(&self, rhs: &Lex) -> bool {
-        self.token == rhs.token && self.value == rhs.value
-    }
-}

lexer/tests/single_tokens.rs

@@ -67,16 +67,24 @@ fn bool_long_false() {
 #[test]
 fn bool_with_spaces() {
     // See issue #12
-    let expected_lex = Lex::new(Token::Bool(false), "#f", 0, 0);
+    let expected_lex = Lex::new(Token::Bool(false), "#f", 0, 2);
     let mut lex = Lexer::new("  #f  ".chars());
     assert_eq!(lex.next(), Some(Ok(expected_lex)));
     assert_eq!(lex.next(), None);
 }
 
 #[test]
-fn integer() {
+fn simple_integers() {
     let mut lex = Lexer::new("23 42".chars());
     assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(23), "23", 0, 0))));
-    assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(42), "42", 0, 0))));
+    assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(42), "42", 0, 3))));
+    assert_eq!(lex.next(), None);
+}
+
+#[test]
+fn integers_in_alternative_bases() {
+    let mut lex = Lexer::new("#x2A #b11001".chars());
+    assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(0x2A), "#x2A", 0, 0))));
+    assert_eq!(lex.next(), Some(Ok(Lex::new(Token::Num(0b11001), "#b11001", 0, 5))));
     assert_eq!(lex.next(), None);
 }