diff --git a/src/handlers.rs b/src/handlers.rs new file mode 100644 index 0000000..6abdd73 --- /dev/null +++ b/src/handlers.rs @@ -0,0 +1,83 @@ +impl crate::Lexer { + pub fn consume_whitespace(&mut self) { + while let Some(c) = self.current_char { + if !c.is_whitespace() { + break; + } + self.advance(); + } + } + + pub fn consume_unknown(&mut self) -> crate::Token { + let unknown = self.current_char.unwrap(); + self.advance(); + crate::Token::Unknown(unknown) + } + + pub fn consume_identifier_or_keyword(&mut self) -> crate::Token { + let mut identifier = String::new(); + while let Some(c) = self.current_char { + if c.is_alphanumeric() || c == '_' { + identifier.push(c); + self.advance(); + } else { + break; + } + } + if crate::is_keyword(&identifier) { + crate::Token::Keyword(identifier) + } else { + crate::Token::Identifier(identifier) + } + } + + pub fn consume_number(&mut self) -> crate::Token { + let mut number = String::new(); + let mut is_float = false; + + while let Some(c) = self.current_char { + if c.is_digit(10) { + number.push(c); + self.advance(); + } else if c == '.' { + if number.is_empty() { + number.push('0'); + } + number.push(c); + is_float = true; + self.advance(); + } else { + break; + } + } + + if is_float { + crate::Token::FloatLiteral(number.parse().unwrap()) + } else { + crate::Token::IntegerLiteral(number.parse().unwrap()) + } + } + + pub fn consume_comment(&mut self) -> crate::Token { + let mut comment = String::new(); + while let Some(c) = self.current_char { + if c == '\n' { + break; + } + comment.push(c); + self.advance(); + } + crate::Token::Comment(comment) + } + + pub fn consume_symbol(&mut self) -> crate::Token { + let symbol = self.current_char.unwrap(); + self.advance(); + crate::Token::Symbol(symbol) + } + pub fn consume_operator(&mut self) -> crate::Token { + let operator = self.current_char.unwrap(); + self.advance(); + crate::Token::Operator(operator.to_string()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 2c43c48..292ad1c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,8 @@ //! # WIP THAT SHIT STILL WONKY AF +mod handlers; + #[derive(Debug, PartialEq)] pub enum Token { /// Something like `float` @@ -102,6 +104,9 @@ impl Lexer { '{' | '}' | '(' | ')' | '#' | ',' | ';' => { tokens.push(self.consume_symbol()); } + '.' => { + tokens.push(self.consume_number()); + } _ => { tokens.push(self.consume_unknown()); @@ -121,88 +126,9 @@ impl Lexer { } } - fn consume_whitespace(&mut self) { - while let Some(c) = self.current_char { - if !c.is_whitespace() { - break; - } - self.advance(); - } - } - - fn consume_unknown(&mut self) -> Token { - let unknown = self.current_char.unwrap(); - self.advance(); - Token::Unknown(unknown) - } - - fn consume_identifier_or_keyword(&mut self) -> Token { - let mut identifier = String::new(); - while let Some(c) = self.current_char { - if c.is_alphanumeric() || c == '_' { - identifier.push(c); - self.advance(); - } else { - break; - } - } - if is_keyword(&identifier) { - Token::Keyword(identifier) - } else { - Token::Identifier(identifier) - } - } - - fn consume_number(&mut self) -> Token { - let mut number = String::new(); - let mut is_float = false; - - while let Some(c) = self.current_char { - if c.is_digit(10) { - number.push(c); - self.advance(); - } else if c == '.' { - number.push(c); - is_float = true; - self.advance(); - } else { - break; - } - } - - if is_float { - Token::FloatLiteral(number.parse().unwrap()) - } else { - Token::IntegerLiteral(number.parse().unwrap()) - } - } - - fn consume_comment(&mut self) -> Token { - let mut comment = String::new(); - while let Some(c) = self.current_char { - if c == '\n' { - break; - } - comment.push(c); - self.advance(); - } - Token::Comment(comment) - } - - fn consume_symbol(&mut self) -> Token { - let symbol = self.current_char.unwrap(); - self.advance(); - Token::Symbol(symbol) - } - fn consume_operator(&mut self) -> Token { - let operator = self.current_char.unwrap(); - self.advance(); - Token::Operator(operator.to_string()) - } - - fn error(&self, message: &str) -> ! { - panic!("Lexer error at position {}: {}", self.position, message); - } + // fn error(&self, message: &str) -> ! { + // panic!("Lexer error at position {}: {}", self.position, message); + // } } fn is_keyword(word: &str) -> bool { @@ -282,11 +208,21 @@ mod tests { #[test] fn float_literal() { init(); - let source_code = "123.45"; + let source_code = "123.4504"; let mut lexer = Lexer::new(source_code); let tokens = lexer.get_tokens(); info!("[FloatLiteral] Tokens: {:#?}", tokens); - assert_eq!(tokens, vec![Token::FloatLiteral(123.45), Token::EOF]); + assert_eq!(tokens, vec![Token::FloatLiteral(123.4504), Token::EOF]); + } + + #[test] + fn float_shorthand() { + init(); + let source_code = ".4504"; + let mut lexer = Lexer::new(source_code); + let tokens = lexer.get_tokens(); + info!("[FloatLiteral Shorthand] Tokens: {:#?}", tokens); + assert_eq!(tokens, vec![Token::FloatLiteral(0.4504), Token::EOF]); } #[test]