/* lexer/src/lib.rs * Eryn Wells */ extern crate sibiltypes; use std::iter::Peekable; use states::{Begin, Resume, StateResult}; mod chars; mod error; mod states; mod token; pub use error::Error; pub use token::{Lex, Token}; pub type Result = std::result::Result; pub struct Lexer where T: Iterator { /// The input stream. input: Peekable, /// Current line number. line: usize, /// Character offset from the start of the input. offset: usize, } impl Lexer where T: Iterator { pub fn new(input: T) -> Lexer { Lexer { input: input.peekable(), line: 0, offset: 0, } } fn next(&mut self) -> Option { let out = self.input.next(); if let Some(c) = out { self.update_offsets(c); } out } fn handle_error(&self, err: Error) { panic!("{}:{}: {}", self.line, self.offset, err.msg()) } fn prepare_offsets(&mut self) { } fn update_offsets(&mut self, c: char) { match c { '\n' => { self.line += 1; self.offset = 0; }, _ => self.offset += 1 } println!("incremented offsets {}:{}", self.line, self.offset); } } impl Iterator for Lexer where T: Iterator { type Item = Result; fn next(&mut self) -> Option { self.prepare_offsets(); let mut token_line = self.line; let mut token_offset = self.offset; println!("beginning token at {}:{}", token_line, token_offset); let mut buffer = String::new(); let mut state: Box = Box::new(Begin::new()); let mut out: Option = None; loop { let peek = self.input.peek().map(char::clone); println!("lexing {:?} in state {:?}, buffer = {:?}", peek, state, buffer); match peek { None => match state.none() { Ok(None) => break, Ok(Some(token)) => { out = Some(Ok(Lex::new(token, &buffer, token_line, token_offset))); break; }, Err(err) => self.handle_error(err) }, Some(c) => { let result = state.lex(c); match result { StateResult::Continue => { buffer.push(c); self.next(); }, StateResult::Advance { to } => { buffer.push(c); self.next(); state = to; }, StateResult::Discard(resume) => { buffer.clear(); state = Box::new(Begin::new()); if resume == Resume::AtNext { self.next(); } token_line = self.line; token_offset = self.offset; }, StateResult::Emit(token, resume) => { if resume == Resume::AtNext { buffer.push(c); self.next(); } out = Some(Ok(Lex::new(token, &buffer, token_line, token_offset))); break; }, StateResult::Fail(err) => self.handle_error(err), } }, } } println!("emitting {:?}", out); out } }