diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 52372cd..bbd6972 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -5,9 +5,11 @@ pub mod token; mod char; mod charset; +mod number; mod str; use self::char::Lexable; +use self::number::NumberBuilder; use self::str::CharAt; use self::str::RelativeIndexable; use self::token::Lex; @@ -15,11 +17,13 @@ use self::token::Token; #[derive(Debug)] enum State { + Comment, Initial, Identifier, Dot, Hash, - Comment, + Number, + NumberDecimal, String, } @@ -29,6 +33,7 @@ pub struct Lexer { forward: usize, line: u32, state: State, + number_builder: NumberBuilder, } impl Lexer { @@ -39,6 +44,7 @@ impl Lexer { forward: 0, line: 1, state: State::Initial, + number_builder: NumberBuilder::new(), } } } @@ -111,6 +117,13 @@ impl Lexer { self.advance(); } + else if c.is_digit(10) { + self.number_builder = NumberBuilder::new(); + self.number_builder.extend_value(c); + self.state = State::Number; + self.advance(); + } + else if c.is_whitespace() { if c.is_newline() { self.handle_newline(); @@ -148,6 +161,12 @@ impl Lexer { *token = Some(Token::Dot); self.retract(); } + else if c.is_digit(10) { + self.number_builder = NumberBuilder::new(); + self.number_builder.extend_decimal_value(c); + self.state = State::NumberDecimal; + self.advance(); + } else { assert!(false, "Invalid token character: '{}'", c); } @@ -167,6 +186,39 @@ impl Lexer { } } + fn state_number(&mut self, c: char, token: &mut Option) { + if c.is_digit(self.number_builder.radix_value()) { + self.number_builder.extend_value(c); + self.advance(); + } + else if c.is_dot() { + self.number_builder.extend_decimal_value(c); + self.state = State::NumberDecimal; + self.advance(); + } + else if c.is_identifier_delimiter() { + *token = Some(Token::Number(self.number_builder.resolve())); + self.retract(); + } + else { + assert!(false, "Invalid token character: '{}'", c); + } + } + + fn state_number_decimal(&mut self, c: char, token: &mut Option) { + if c.is_digit(self.number_builder.radix_value()) { + self.number_builder.extend_decimal_value(c); + self.advance(); + } + else if c.is_identifier_delimiter() { + *token = Some(Token::Number(self.number_builder.resolve())); + self.retract(); + } + else { + assert!(false, "Invalid token character: '{}'", c); + } + } + fn state_string(&mut self, c: char, token: &mut Option) { self.advance(); if c.is_string_quote() { @@ -209,6 +261,8 @@ impl Iterator for Lexer { State::Identifier => self.state_identifier(c, &mut token), State::Dot => self.state_dot(c, &mut token), State::Hash => self.state_hash(c, &mut token), + State::Number => self.state_number(c, &mut token), + State::NumberDecimal => self.state_number_decimal(c, &mut token), State::String => self.state_string(c, &mut token), State::Comment => self.state_comment(c, &mut token), } @@ -230,6 +284,7 @@ impl Iterator for Lexer { mod tests { use std::iter::Iterator; use super::*; + use super::number::*; use super::token::*; #[test] @@ -274,6 +329,13 @@ mod tests { check_single_token("\"abc\"", Token::String(String::from("\"abc\""))); } + #[test] + fn lexer_finds_numbers() { + check_single_token("34", Token::Number(Number::new(34.0))); + check_single_token(".34", Token::Number(Number::new(0.34))); + check_single_token("0.34", Token::Number(Number::new(0.34))); + } + fn check_single_token(input: &str, expected: Token) { let mut lexer = Lexer::new(input); assert_next_token(&mut lexer, &expected); diff --git a/src/lexer/number.rs b/src/lexer/number.rs index 0134fbb..8e24480 100644 --- a/src/lexer/number.rs +++ b/src/lexer/number.rs @@ -2,8 +2,13 @@ * Eryn Wells */ -struct Number { - value: f64, +#[derive(PartialEq, Debug)] +pub struct Number { value: f64, } + +impl Number { + pub fn new(value: f64) -> Number { + Number { value: value } + } } #[derive(Debug)] @@ -14,6 +19,7 @@ pub struct NumberBuilder { exact: bool, radix: Radix, value: f64, + point: u16, } impl NumberBuilder { @@ -22,6 +28,7 @@ impl NumberBuilder { exact: false, radix: Radix::Dec, value: 0.0, + point: 0, } } @@ -37,7 +44,12 @@ impl NumberBuilder { pub fn resolve(&self) -> Number { // TODO: Convert fields to Number type. - Number { value: 0.0 } + let value = if self.point == 0 { + self.value + } else { + self.value / (self.point * 10) as f64 + }; + Number { value: value } } pub fn extend_value<'a>(&'a mut self, digit: char) -> &'a mut Self { @@ -50,6 +62,12 @@ impl NumberBuilder { self } + pub fn extend_decimal_value<'a>(&'a mut self, digit: char) -> &'a mut Self { + self.extend_value(digit); + self.point += 1; + self + } + pub fn radix_value(&self) -> u32 { self.radix.value() as u32 } diff --git a/src/lexer/token.rs b/src/lexer/token.rs index c56b6ca..431c5d9 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -2,17 +2,19 @@ * Eryn Wells */ -#[derive(Debug)] -#[derive(PartialEq)] +use lexer::number::Number; + +#[derive(PartialEq, Debug)] pub enum Token { - LeftParen(String), - LeftVectorParen, - RightParen(String), + Boolean(bool), + Comment(String), Dot, Identifier(String), - Boolean(bool), + LeftParen(String), + LeftVectorParen, + Number(Number), + RightParen(String), String(String), - Comment(String), } /// A Lex is a Token extracted from a specific position in an input. It contains useful information about the token's