From 75b210619aa7d43538ffb577c2b60b6762660f75 Mon Sep 17 00:00:00 2001
From: Amneesh Singh <natto@weirdnatto.in>
Date: Sun, 30 Jul 2023 18:43:57 +0530
Subject: [PATCH] lexer: add exponents to floating literals

Signed-off-by: Amneesh Singh <natto@weirdnatto.in>
---
 src/lexer.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 54 insertions(+), 7 deletions(-)

diff --git a/src/lexer.rs b/src/lexer.rs
index d68fe1b..7f028a2 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -14,11 +14,9 @@ pub enum TokenLiteral {
 }
 
 /// All token symbols
-///
-/// TODO: Maybe add *
 #[derive(Debug, PartialEq)]
 pub enum TokenSymbol {
-    // operators
+    // arithmetic
     Plus,
     Minus,
     Star,
@@ -230,14 +228,19 @@ impl<'a> Lexer<'a> {
         let mut is_float: bool = false;
         while let Some(c) = self.peek() {
             match c {
-                '0'..='9' => {}
+                '0'..='9' => {},
                 '.' => {
                     if is_float {
-                        self.error("Multiple decimals encountered");
+                        self.error("multiple decimals encountered");
                         return self.new_token(TokenKind::Invalid);
                     }
                     is_float = true;
                 }
+                'e' => {
+                    self.next();
+                    is_float = true;
+                    break;
+                }
                 _ => break,
             }
             self.next();
@@ -259,6 +262,11 @@ impl<'a> Lexer<'a> {
             return self.new_token(TokenKind::Invalid);
         }
 
+        if self.peek() != Some(&'\'') {
+            self.error("Expected '");
+            return self.new_token(TokenKind::Invalid);
+        }
+
         // skip '
         self.next();
 
@@ -441,7 +449,7 @@ fn test_peek_next() {
 }
 
 #[test]
-fn test_tokens_1() {
+fn test_tokens() {
     let mut lexer = Lexer::new("let test02 = 4 << 1");
 
     use TokenKind::*;
@@ -501,9 +509,48 @@ fn test_tokens_2() {
 }
 
 #[test]
-fn test_tokens_3() {
+fn test_tokens_eof() {
     let mut lexer = Lexer::new("");
 
     assert_eq!(lexer.peek_token().kind, TokenKind::Eof);
     assert_eq!(lexer.next_token().kind, TokenKind::Eof);
 }
+
+#[test]
+fn test_tokens_numeric() {
+    let mut lexer = Lexer::new("3342");
+
+    let token = lexer.next_token();
+    assert_eq!(token.kind, TokenKind::Literal(TokenLiteral::Int));
+    assert_eq!(*token.val, *"3342");
+
+    assert_eq!(lexer.next_token().kind, TokenKind::Eof);
+}
+
+#[test]
+fn test_tokens_numeric_2() {
+    let mut lexer = Lexer::new("334.2e");
+
+    let token = lexer.next_token();
+    assert_eq!(token.kind, TokenKind::Literal(TokenLiteral::Float));
+    assert_eq!(*token.val, *"334.2e");
+
+    assert_eq!(lexer.next_token().kind, TokenKind::Eof);
+}
+
+#[test]
+fn test_tokens_numeric_3() {
+    let mut lexer = Lexer::new("334.2e-5");
+
+    let mut token = lexer.next_token();
+    assert_eq!(token.kind, TokenKind::Literal(TokenLiteral::Float));
+    assert_eq!(*token.val, *"334.2e");
+    
+    assert_eq!(lexer.next_token().kind, TokenKind::Symbol(TokenSymbol::Minus));
+
+    token = lexer.next_token();
+    assert_eq!(token.kind, TokenKind::Literal(TokenLiteral::Int));
+    assert_eq!(*token.val, *"5");
+
+    assert_eq!(lexer.next_token().kind, TokenKind::Eof);
+}