From 5a0656fbe726ad3ca916af116fabdc9be3695b69 Mon Sep 17 00:00:00 2001 From: xqtc Date: Thu, 1 Aug 2024 01:28:14 +0200 Subject: [PATCH] Support swizzling; Some stuff for float handling --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + flake.nix | 2 +- src/ast/ast.rs | 34 ++++++++++++++++++++++++++++++ src/ast/mod.rs | 1 + src/lex/handlers.rs | 50 +++++++++++++++++++++++++++++++++++++++++---- src/lex/lexer.rs | 8 ++++---- src/lex/util.rs | 6 ++++++ src/lib.rs | 28 +++++++++++++++++++++++++ src/tokens.rs | 1 + 10 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 src/ast/ast.rs create mode 100644 src/ast/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 1e8cd8a..de9e8f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,6 +94,7 @@ name = "glsl-lexer" version = "0.1.0" dependencies = [ "env_logger", + "lazy_static", "log", ] @@ -109,6 +110,12 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "log" version = "0.4.22" diff --git a/Cargo.toml b/Cargo.toml index 3827790..e87b72e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,4 +5,5 @@ edition = "2021" [dependencies] env_logger = "0.11.3" +lazy_static = "1.5.0" log = "0.4.22" diff --git a/flake.nix b/flake.nix index 9692ffc..49a0a5a 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,7 @@ ]; rust-toolchain = pkgs.symlinkJoin { name = "rust-toolchain"; - paths = [ pkgs.rustc pkgs.cargo pkgs.cargo-watch pkgs.clippy pkgs.rust-analyzer pkgs.rustPlatform.rustcSrc ]; + paths = [ pkgs.rustc pkgs.cargo pkgs.cargo-watch pkgs.cargo-nextest pkgs.clippy pkgs.rust-analyzer pkgs.rustPlatform.rustcSrc ]; }; in { diff --git a/src/ast/ast.rs b/src/ast/ast.rs new file mode 100644 index 0000000..1ff6ded --- /dev/null +++ b/src/ast/ast.rs @@ -0,0 +1,34 @@ +use crate::{lex::lexer::Lexer, tokens::Token}; + +struct AST { + nodes: Vec, +} + +struct Node { + token: Token, + children: Vec, +} + +impl Node { + pub fn new(token: Token) -> Self { + Node { + token, + children: Vec::new(), + } + } + + pub fn add_child(&mut self, node: Node) { + self.children.push(node); + } +} + +impl AST { + pub fn new() -> Self { + AST { nodes: Vec::new() } + } + + pub fn add_node(&mut self, token: Token) { + let node = Node::new(token); + self.nodes.push(node); + } +} diff --git a/src/ast/mod.rs b/src/ast/mod.rs new file mode 100644 index 0000000..c49b124 --- /dev/null +++ b/src/ast/mod.rs @@ -0,0 +1 @@ +mod ast; diff --git a/src/lex/handlers.rs b/src/lex/handlers.rs index 2b2b1dd..4a5c52e 100644 --- a/src/lex/handlers.rs +++ b/src/lex/handlers.rs @@ -1,3 +1,5 @@ +use core::panic; + use crate::lex::lexer::Lexer; impl Lexer { pub fn consume_whitespace(&mut self) { @@ -33,19 +35,59 @@ impl Lexer { pub fn consume_number(&mut self) -> crate::tokens::Token { let mut number = String::new(); let mut is_float = false; + let mut is_swizzle = false; while let Some(c) = self.current_char { - if c.is_numeric() { + if c.is_digit(10) + /* && self.peek().unwrap_or_else(|| ' ') == 'f' */ + { number.push(c); self.advance(); - } else if c == '.' || c == 'e' { - is_float = true; + } else if c == '.' || c == 'e' || c == 'f' { + //////////////////////////////////// + //ALLES HIER DRIN IST NICHT SCHÖN// + //////////////////////////////////// + + match self.peek().unwrap_or_else(|| ' ') { + 'x' | 'y' | 'z' | 'w' | 'r' | 'g' | 'b' | 'a' => { + is_swizzle = true; + number.push(c); + self.advance(); + } + '0'..='9' | 'e' => { + is_float = true; + number.push(c); + self.advance(); + } + _ => {} + } + if c == 'f' { + is_float = true; + number.push(c); + self.advance(); + dbg!(&number); + break; + } + } else if c.is_alphabetic() { + if c == '.' && self.peek().unwrap_or_else(|| ' ') == 'e' { + is_float = true; + self.advance(); + dbg!("break in alphabetic"); + break; + } + is_swizzle = true; number.push(c); - self.advance() + self.advance(); + } else { } } if is_float { return crate::tokens::Token::FLOATCONSTANT(number); } + if is_swizzle { + let split: Vec<&str> = number.split('.').collect(); + let ident2 = split[1]; + return crate::tokens::Token::Swizzle(vec![".".to_string(), ident2.to_string()]); + } crate::tokens::Token::INTCONSTANT(number) } diff --git a/src/lex/lexer.rs b/src/lex/lexer.rs index 448e828..d27287f 100644 --- a/src/lex/lexer.rs +++ b/src/lex/lexer.rs @@ -29,10 +29,10 @@ impl Lexer { input: input.chars().collect(), position: 0, current_char: None, - keywords: make_keywords(), - symbols: make_symbols(), + keywords: crate::lex::util::KEYWORDS.clone(), + symbols: crate::lex::util::SYMBOLS.clone(), }; - dbg!("{}", &lexer.keywords); + // dbg!("{}", &lexer.keywords); lexer.current_char = if lexer.position < lexer.input.len() { Some(lexer.input[lexer.position]) } else { @@ -85,7 +85,7 @@ impl Lexer { tokens.push(Token::Whitespace); } else if c.is_alphabetic() || c == '_' { tokens.push(self.consume_identifier_or_keyword()); - } else if c.is_ascii_digit() { + } else if c.is_digit(10) { tokens.push(self.consume_number()); } else if c == '/' && self.peek() == Some('/') { tokens.push(self.consume_comment()); diff --git a/src/lex/util.rs b/src/lex/util.rs index b87a0c9..7fcb5a4 100644 --- a/src/lex/util.rs +++ b/src/lex/util.rs @@ -1,7 +1,13 @@ +use lazy_static::lazy_static; use std::collections::HashMap; use crate::tokens::{Image, Material, Sampler, Token, Vector}; +lazy_static! { + pub static ref SYMBOLS: HashMap = make_symbols(); + pub static ref KEYWORDS: HashMap<&'static str, Token> = make_keywords(); +} + pub fn make_symbols() -> HashMap { let mut symbols = HashMap::new(); symbols.insert('{', Token::LEFT_BRACE); diff --git a/src/lib.rs b/src/lib.rs index 2aa22ef..8e9dba4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ #![allow(dead_code)] #![allow(non_camel_case_types)] +mod ast; pub mod lex; mod tokens; @@ -27,6 +28,17 @@ mod tests { use super::lex::lexer::Lexer; use super::tokens::{Image, Token}; + #[test] + fn float_with_f_aggot() { + let source = "5f"; + let mut lexer = Lexer::new(source); + let tokens = Lexer::get_tokens(&mut lexer); + assert_eq!( + tokens, + vec![Token::FLOATCONSTANT("5f".to_string()), Token::EOF].into() + ) + } + #[test] fn keyword() { let source = "image1D"; @@ -134,6 +146,22 @@ mod tests { vec![Token::INTCONSTANT("12345".to_string()), Token::EOF].into() ); } + + #[test] + fn swizzling() { + let source_code = "abcd.xyz"; + let mut lexer = Lexer::new(source_code); + let tokens = lexer.get_tokens(); + assert_eq!( + tokens, + vec![ + Token::IDENTIFIER("abcd".to_string()), + Token::Swizzle(vec![".".to_string(), "xyz".to_string()]), + Token::EOF + ] + .into() + ); + } } // #[cfg(test)] // mod tests { diff --git a/src/tokens.rs b/src/tokens.rs index 99ae276..c6e2049 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -4,6 +4,7 @@ pub enum Token { EOF, Whitespace, + Swizzle(Vec), Unknown(String), Comment(String), CONST,