From c2966ce9738e4216d75e1aef5d673510fb63d143 Mon Sep 17 00:00:00 2001
From: xqtc <xqtc@tutanota.com>
Date: Wed, 24 Jul 2024 21:05:17 +0200
Subject: [PATCH] Small changes to keyword handling; Comment consumption

---
 src/lex/handlers.rs | 15 +++++++++++++--
 src/lex/util.rs     |  7 ++-----
 src/lib.rs          | 33 ++++++++++++++++-----------------
 src/tokens.rs       |  1 +
 4 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/src/lex/handlers.rs b/src/lex/handlers.rs
index 178a950..03bfb64 100644
--- a/src/lex/handlers.rs
+++ b/src/lex/handlers.rs
@@ -23,7 +23,10 @@ impl Lexer {
                 break;
             }
         }
-        self.is_keyword(&word).unwrap()
+        match self.is_keyword(&word) {
+            Some(token) => token,
+            None => crate::tokens::Token::IDENTIFIER(word),
+        }
     }
 
     pub fn consume_number(&mut self) -> crate::tokens::Token {
@@ -31,7 +34,15 @@ impl Lexer {
     }
 
     pub fn consume_comment(&mut self) -> crate::tokens::Token {
-        todo!()
+        let mut comment = String::new();
+        while let Some(c) = self.current_char {
+            if c == '\n' {
+                break;
+            }
+            comment.push(c);
+            self.advance();
+        }
+        crate::tokens::Token::Comment(comment)
     }
 
     pub fn consume_symbol(&mut self) -> crate::tokens::Token {
diff --git a/src/lex/util.rs b/src/lex/util.rs
index e36a813..ec57bba 100644
--- a/src/lex/util.rs
+++ b/src/lex/util.rs
@@ -1,4 +1,3 @@
-
 use crate::tokens::{Image, Material, Sampler, Token, Vector};
 
 pub fn populate_tokens(lexer: &mut crate::lex::lexer::Lexer) {
@@ -373,10 +372,8 @@ impl crate::lex::lexer::Lexer {
     pub fn is_keyword(&mut self, word: &str) -> Option<Token> {
         let token = self.keywords.get(word);
         if let Some(token) = token {
-            Some(token.clone())
-        } else {
-            // TODO: Check if word is an identifier
-            Some(Token::IDENTIFIER(word.to_string()))
+            return Some(token.clone());
         }
+        None
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 4819350..976e41d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -56,6 +56,22 @@ mod tests {
         let tokens = lexer.get_tokens();
         assert_eq!(tokens, vec![Token::UNIFORM, Token::EOF].into());
     }
+
+    #[test]
+    fn test_single_line_comment() {
+        let source = "// This is a comment\n";
+        let mut lexer = Lexer::new(source);
+        let tokens = lexer.get_tokens();
+        assert_eq!(
+            tokens,
+            vec![
+                Token::Comment("// This is a comment".to_string()),
+                Token::Whitespace,
+                Token::EOF,
+            ]
+            .into()
+        );
+    }
 }
 // #[cfg(test)]
 // mod tests {
@@ -164,23 +180,6 @@ mod tests {
 //         );
 //     }
 //
-//     #[test]
-//     fn test_single_line_comment() {
-//         init();
-//         let source = "// This is a comment\n";
-//         let mut lexer = Lexer::new(source);
-//         let tokens = lexer.get_tokens();
-//         info!("[Comment] Tokens: {:#?}", tokens);
-//         assert_eq!(
-//             tokens,
-//             vec![
-//                 Token::Comment("// This is a comment".to_string()),
-//                 Token::Whitespace,
-//                 Token::EOF,
-//             ]
-//             .into()
-//         );
-//     }
 //
 //     // I hope that does it. Writing this test was pain.
 //     #[test]
diff --git a/src/tokens.rs b/src/tokens.rs
index a14ac14..3e0583f 100644
--- a/src/tokens.rs
+++ b/src/tokens.rs
@@ -4,6 +4,7 @@
 pub enum Token {
     EOF,
     Whitespace,
+    Comment(String),
     CONST,
     BOOL,
     FLOAT,