revive project

Signed-off-by: Amneesh Singh <natto@weirdnatto.in>

.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  checks:
+    name: Checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v20
+        with:
+          extra_nix_config: |
+            auto-optimise-store = true
+            experimental-features = nix-command flakes
+      - uses: cachix/cachix-action@v12
+        with:
+          name: pain
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: fmt check
+        run: nix build .#checks.fmt -L
+
+      - name: clippy check
+        run: nix build .#checks.clippy -L
+        
+      - name: nextest check
+        run: nix build .#checks.nextest -L
+
+      - name: doc tests
+        run: nix build .#checks.doc -L
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: [ checks ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v20
+        with:
+          extra_nix_config: |
+            auto-optimise-store = true
+            experimental-features = nix-command flakes
+      - uses: cachix/cachix-action@v12
+        with:
+          name: pain
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: fmt check
+        run: nix build .#tricc -L

flake.lock

@@ -10,11 +10,11 @@
         "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1680584903,
+        "lastModified": 1688772518,
-        "narHash": "sha256-uraq+D3jcLzw/UVk0xMHcnfILfIMa0DLrtAEq2nNlxU=",
+        "narHash": "sha256-ol7gZxwvgLnxNSZwFTDJJ49xVY5teaSvF7lzlo3YQfM=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "65d3f6a3970cd46bef5eedfd458300f72c56b3c5",
+        "rev": "8b08e96c9af8c6e3a2b69af5a7fa168750fcf88e",
         "type": "github"
       },
       "original": {
@@ -40,12 +40,15 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1678901627,
+        "lastModified": 1687709756,
-        "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
+        "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
+        "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
         "type": "github"
       },
       "original": {
@@ -55,12 +58,15 @@
       }
     },
     "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1678901627,
+        "lastModified": 1689068808,
-        "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
         "type": "github"
       },
       "original": {
@@ -70,12 +76,15 @@
       }
     },
     "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_3"
+      },
       "locked": {
-        "lastModified": 1659877975,
+        "lastModified": 1681202837,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
         "type": "github"
       },
       "original": {
@@ -86,11 +95,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1680724564,
+        "lastModified": 1688392541,
-        "narHash": "sha256-eeUUGOTKTelYKDbUxKs0V7GUa186L2fym7jM2QQ4Oss=",
+        "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36adaa6aaa6b03e59102df0c1b12cdc3f23fd112",
+        "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
         "type": "github"
       },
       "original": {
@@ -120,11 +129,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1680488274,
+        "lastModified": 1688351637,
-        "narHash": "sha256-0vYMrZDdokVmPQQXtFpnqA2wEgCCUXf5a3dDuDVshn0=",
+        "narHash": "sha256-CLTufJ29VxNOIZ8UTg0lepsn3X03AmopmaLTTeHDCL4=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "7ec2ff598a172c6e8584457167575b3a1a5d80d8",
+        "rev": "f9b92316727af9e6c7fee4a761242f7f46880329",
         "type": "github"
       },
       "original": {
@@ -141,11 +150,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1680660688,
+        "lastModified": 1690252178,
-        "narHash": "sha256-XeQTCxWBR0Ai1VMzI5ZXYpA2lu1F8FzZKjw8RtByZOg=",
+        "narHash": "sha256-9oEz822bvbHobfCUjJLDor2BqW3I5tycIauzDlzOALY=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "2f40052be98347b479c820c00fb2fc1d87b3aa28",
+        "rev": "8d64353ca827002fb8459e44d49116c78d868eba",
         "type": "github"
       },
       "original": {
@@ -153,6 +162,51 @@
         "repo": "rust-overlay",
         "type": "github"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -38,25 +38,34 @@
 
         tricc = craneLib.buildPackage (commonArgs // {
           inherit cargoArtifacts;
+          doCheck = false;
         });
       in
       {
-        checks = {
-          inherit tricc;
-
-          clippy = craneLib.cargoClippy (commonArgs // {
-            inherit cargoArtifacts;
-            cargoClippyExtraArgs = "--all-targets -- --deny warnings";
-          });
-
-          fmt = craneLib.cargoFmt {
-            inherit src;
-          };
-        };
-
         packages = {
           inherit tricc;
           default = tricc;
+
+          # not using flake checks to run them individually
+          checks = {
+            clippy = craneLib.cargoClippy (commonArgs // {
+              inherit cargoArtifacts;
+            });
+
+            fmt = craneLib.cargoFmt {
+              inherit src;
+            };
+
+            doc = craneLib.cargoDoc (commonArgs // {
+              inherit cargoArtifacts;
+            });
+
+            nextest = craneLib.cargoNextest (commonArgs // {
+              inherit cargoArtifacts;
+              partitions = 1;
+              partitionType = "count";
+            });
+          };
         };
 
         devShells.default = pkgs.mkShell {

rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-04-01"
+channel = "nightly-2023-07-15"
 components = [ "rustfmt", "clippy", "rust-analyzer", "rust-src" ]

rustfmt.toml

@@ -0,0 +1,5 @@
+comment_width = 99
+format_code_in_doc_comments = true
+imports_granularity = "Module"
+imports_layout = "Vertical"
+wrap_comments = true

src/args.rs

@@ -4,7 +4,7 @@ use std::process::exit;
 const VERSION: &str = env!("CARGO_PKG_VERSION");
 const CRATE: &str = env!("CARGO_CRATE_NAME");
 
-// naive argument handling
+/// A naive argument handler
 #[derive(Default)]
 pub struct Args {
     version: bool,
@@ -12,13 +12,12 @@ pub struct Args {
 }
 
 impl Args {
+    /// Creates a new [`Args`] instance
     pub fn new() -> Args {
-        Args {
+        Args::default()
-            version: false,
-            file: None,
-        }
     }
 
+    /// Checks for various arguments
     pub fn handle(&mut self) {
         let args: Vec<String> = env::args().collect();
 
@@ -53,6 +52,8 @@ impl Args {
         }
     }
 
+    /// Fetches the file from the arguments.
+    /// Panics if there is no file in the arguments
     #[inline]
     pub fn get_file(self) -> String {
         self.file.expect("no file supplied!")

src/lexer.rs

@@ -1,13 +1,22 @@
-use std::{iter, str};
+use std::collections::VecDeque;
+use std::iter::Peekable;
+use std::rc::Rc;
+use std::str;
 
-#[derive(Debug)]
+/// All token literals
+///
+/// TODO: Add string
+#[derive(Debug, PartialEq)]
 pub enum TokenLiteral {
     Int,
     Float,
     Char,
 }
 
-#[derive(Debug)]
+/// All token symbols
+///
+/// TODO: Maybe add *
+#[derive(Debug, PartialEq)]
 pub enum TokenSymbol {
     // operators
     Plus,
@@ -58,10 +67,16 @@ pub enum TokenSymbol {
     Hash,
 }
 
-#[derive(Debug)]
+/// All token keywod
+#[derive(Debug, PartialEq)]
 pub enum TokenKeyword {
-    Let,
+    // parents
     Fn,
+    Class,
+    Module,
+
+    // statements
+    Let,
     Ret,
 
     // conditionals
@@ -69,10 +84,10 @@ pub enum TokenKeyword {
     Else,
     Elif,
 
-    // loops
+    // control flow
-    While,
+    Loop,
-    Do,
+    Break,
-    For,
+    Continue,
 
     // primitives
     Int,
@@ -80,7 +95,10 @@ pub enum TokenKeyword {
     Char,
 }
 
-#[derive(Debug)]
+/// All token delimiters
+///
+/// TODO: Maybe add \[ and \]
+#[derive(Debug, PartialEq)]
 pub enum TokenDelimiter {
     BraceOpen,
     BraceClose,
@@ -88,31 +106,70 @@ pub enum TokenDelimiter {
     ParenClose,
 }
 
-#[derive(Debug)]
+/// All tokens
-pub enum Token<'a> {
+#[derive(Debug, PartialEq)]
+pub enum TokenKind {
     Newline,
-    Literal(TokenLiteral, &'a str),
+    Eof,
+    Literal(TokenLiteral),
     Symbol(TokenSymbol),
     Keyword(TokenKeyword),
     Delimiter(TokenDelimiter),
-    Identifier(&'a str),
+    Identifier,
+    Invalid,
+}
+
+#[derive(Debug)]
+pub struct Token {
+    pub kind: TokenKind,
+    /// Holds the reference to the tokenized string
+    ///
+    /// For example, if `kind` is of type [`TokenKind::Identifier`], this would contain the value
+    /// of that identifier
+    pub val: Rc<str>,
 }
 
 pub struct Lexer<'a> {
-    file: &'a str,
+    /// The entire text to be tokenized
     text: &'a str,
-    chars: iter::Peekable<str::Chars<'a>>,
+    /// A peekable iterate for `text`
-    line: usize,
+    chars: Peekable<str::Chars<'a>>,
-    start: usize,
+    /// A peekable double ended queue for the tokens
+    tokens: VecDeque<Token>,
+    /// Current line number
+    pub line: usize,
+    /// Start character index for the current token
+    pub start: usize,
+    /// End character index for the current token
     end: usize,
 }
 
 impl<'a> Lexer<'a> {
-    pub fn new(file: &'a str, contents: &'a str) -> Lexer<'a> {
+    /// Creates a new [`Lexer`] instance with the provided content.
+    ///
+    /// The `Lexer` is responsible for tokenizing the given text, making it easier to
+    /// perform various parsing operations.
+    ///
+    /// # Arguments
+    ///
+    /// * `content`: The text to tokenize.
+    ///
+    /// # Returns
+    ///
+    /// A new instance of `Lexer` initialized with the provided `content`.
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use tricc::lexer::Lexer;
+    ///
+    /// let lexer = Lexer::new("let example: int = 4");
+    /// ```
+    pub fn new(content: &'a str) -> Self {
         Lexer {
-            file,
+            text: content,
-            text: contents,
+            chars: content.chars().peekable(),
-            chars: contents.chars().peekable(),
+            tokens: VecDeque::new(),
             line: 1,
             start: 0,
             end: 0,
@@ -120,49 +177,63 @@ impl<'a> Lexer<'a> {
     }
 
     #[inline]
-    fn error(&self) {
+    fn new_token(&self, kind: TokenKind) -> Token {
-        eprintln!("error lexing \"{}:{}:{}\"", self.file, self.line, self.end);
+        Token {
+            kind,
+            val: Rc::from(&self.text[self.start..self.end]),
+        }
     }
 
+    #[inline]
+    fn error(&self, msg: &str) {
+        eprintln!("Lexer: {}, at \"{}:{}\"", msg, self.line, self.end);
+    }
+
+    #[inline]
+    fn peek(&mut self) -> Option<&char> {
+        self.chars.peek()
+    }
+
+    #[inline]
     fn next(&mut self) -> Option<char> {
         self.end += 1;
         self.chars.next()
     }
 
-    fn skip(&mut self, c: char) {
+    fn skip_whitespace(&mut self) {
-        if self.next() != Some(c) {
+        let mut ignore_nl: bool = false;
-            self.error();
-            panic!("expected {}", c);
-        }
-    }
 
-    fn escape_newline(&mut self) {
+        while let Some(c) = self.peek() {
-        while let Some(c) = self.chars.peek() {
             match c {
                 '\r' | '\t' | ' ' => {
                     self.next();
                 }
                 '\n' => {
-                    self.next();
+                    if ignore_nl {
-                    break;
+                        ignore_nl = false;
+                        self.next();
+                    } else {
+                        break;
+                    }
                 }
-                _ => {
+                '\\' => {
-                    self.error();
+                    self.next();
-                    panic!("expected newline");
+                    ignore_nl = true;
-                },
+                }
+                _ => break,
             }
         }
     }
 
-    fn get_numeric(&mut self) -> Token<'a> {
+    fn get_numeric(&mut self) -> Token {
         let mut is_float: bool = false;
-        while let Some(c) = self.chars.peek() {
+        while let Some(c) = self.peek() {
             match c {
                 '0'..='9' => {}
                 '.' => {
                     if is_float {
-                        self.error();
+                        self.error("Multiple decimals encountered");
-                        panic!("multiple decimals encountered")
+                        return self.new_token(TokenKind::Invalid);
                     }
                     is_float = true;
                 }
@@ -171,62 +242,78 @@ impl<'a> Lexer<'a> {
             self.next();
         }
 
-        Token::Literal(
+        self.new_token(TokenKind::Literal(if is_float {
-            if is_float {
+            TokenLiteral::Float
-                TokenLiteral::Float
+        } else {
-            } else {
+            TokenLiteral::Int
-                TokenLiteral::Int
+        }))
-            },
-            &self.text[self.start..self.end],
-        )
     }
 
-    fn get_char(&mut self) -> Token<'a> {
+    fn get_char(&mut self) -> Token {
-        self.skip('\'');
+        // skip '
+        self.next();
 
         if matches!(self.next(), Some('\'') | None) {
-            self.error();
+            self.error("Expected character literal");
-            panic!("A character literal cannot be empty");
+            return self.new_token(TokenKind::Invalid);
         }
 
-        self.skip('\'');
+        // skip '
+        self.next();
 
-        Token::Literal(TokenLiteral::Char, &self.text[self.start + 1..self.end - 1])
+        self.new_token(TokenKind::Literal(TokenLiteral::Char))
     }
 
-    fn get_delimiter(&mut self) -> Token<'a> {
+    fn get_alphanumeric(&mut self) -> Token {
-        use Token::Delimiter;
+        while let Some(c) = self.peek() {
-        use TokenDelimiter::*;
+            match c {
-
+                'a'..='z' | 'A'..='Z' | '0'..='9' => {}
-        match self.next() {
+                _ => break,
-            Some(c) => match c {
-                '{' => Delimiter(BraceOpen),
-                '}' => Delimiter(BraceClose),
-                '(' => Delimiter(ParenOpen),
-                ')' => Delimiter(ParenClose),
-                _ => {
-                    self.error();
-                    panic!("expected delimiter");
-                }
-            },
-            None => {
-                self.error();
-                panic!("expected delimiter");
             }
+            self.next();
         }
+
+        use TokenKeyword::*;
+        use TokenKind::Keyword;
+
+        self.new_token(match &self.text[self.start..self.end] {
+            "fn" => Keyword(Fn),
+            "class" => Keyword(Class),
+            "module" => Keyword(Module),
+            "let" => Keyword(Let),
+            "ret" => Keyword(Ret),
+            "if" => Keyword(If),
+            "else" => Keyword(Else),
+            "elif" => Keyword(Elif),
+            "loop" => Keyword(Loop),
+            "break" => Keyword(Break),
+            "continue" => Keyword(Continue),
+            "int" => Keyword(Int),
+            "float" => Keyword(Float),
+            "char" => Keyword(Char),
+            _ => TokenKind::Identifier,
+        })
     }
 
-    fn get_symbol(&mut self) -> Token<'a> {
+    fn get_symbol(&mut self) -> Token {
-        use Token::Symbol;
+        let c = self.next().unwrap();
+
+        use TokenDelimiter::*;
+        use TokenKind::{
+            Delimiter,
+            Symbol,
+        };
+        use TokenSymbol::*;
 
         // handle +, +=, -, -=, *, *=, /, /=, %, %=, ^, ^=, !, !=
         macro_rules! token_symbol_eq {
             ($a:expr, $b:expr) => {
-                if self.chars.peek() == Some(&'=') {
+                match self.peek() {
-                    self.next();
+                    Some('=') => {
-                    Symbol($b)
+                        self.next();
-                } else {
+                        Symbol($b)
-                    Symbol($a)
+                    }
+                    _ => Symbol($a),
                 }
             };
         }
@@ -234,7 +321,7 @@ impl<'a> Lexer<'a> {
         // handle &, |, ||, &&, &=, |=
         macro_rules! token_symbol_logical {
             ($a:expr, $b:expr, $c:expr, $d:expr) => {
-                match self.chars.peek() {
+                match self.peek() {
                     Some('=') => {
                         self.next();
                         Symbol($c)
@@ -251,7 +338,7 @@ impl<'a> Lexer<'a> {
         // handle <, <=, >, >=, <<, >>, <<=, >>=
         macro_rules! token_symbol_compare {
             ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => {
-                match self.chars.peek() {
+                match self.peek() {
                     Some('=') => {
                         self.next();
                         Symbol($d)
@@ -265,99 +352,155 @@ impl<'a> Lexer<'a> {
             };
         }
 
-        use TokenSymbol::*;
+        let typ = match c {
-
+            '{' => Delimiter(BraceOpen),
-        match self.next() {
+            '}' => Delimiter(BraceClose),
-            Some(c) => match c {
+            '(' => Delimiter(ParenOpen),
-                '+' => token_symbol_eq!(Plus, PlusEq),
+            ')' => Delimiter(ParenClose),
-                '-' => token_symbol_eq!(Minus, MinusEq),
+            '+' => token_symbol_eq!(Plus, PlusEq),
-                '*' => token_symbol_eq!(Star, StarEq),
+            '-' => token_symbol_eq!(Minus, MinusEq),
-                '/' => token_symbol_eq!(Slash, SlashEq),
+            '*' => token_symbol_eq!(Star, StarEq),
-                '%' => token_symbol_eq!(Percent, PercentEq),
+            '/' => token_symbol_eq!(Slash, SlashEq),
-                '^' => token_symbol_eq!(Caret, CaretEq),
+            '%' => token_symbol_eq!(Percent, PercentEq),
-                '!' => token_symbol_eq!(Not, Ne),
+            '^' => token_symbol_eq!(Caret, CaretEq),
-                '=' => token_symbol_eq!(Eq, EqEq),
+            '!' => token_symbol_eq!(Not, Ne),
-                '&' => token_symbol_logical!(And, AndAnd, AndEq, '&'),
+            '=' => token_symbol_eq!(Eq, EqEq),
-                '|' => token_symbol_logical!(Or, OrOr, OrEq, '|'),
+            '&' => token_symbol_logical!(And, AndAnd, AndEq, '&'),
-                '<' => token_symbol_compare!(Lt, Shl, ShlEq, LtEq, '<'),
+            '|' => token_symbol_logical!(Or, OrOr, OrEq, '|'),
-                '>' => token_symbol_compare!(Gt, Shr, ShrEq, GtEq, '>'),
+            '<' => token_symbol_compare!(Lt, Shl, ShlEq, LtEq, '<'),
-                '~' => Symbol(Tilde),
+            '>' => token_symbol_compare!(Gt, Shr, ShrEq, GtEq, '>'),
-                ':' => Symbol(Colon),
+            '~' => Symbol(Tilde),
-                '.' => Symbol(Dot),
+            ':' => Symbol(Colon),
-                '#' => Symbol(Hash),
+            '.' => Symbol(Dot),
-                _ => {
+            '#' => Symbol(Hash),
-                    self.error();
+            _ => {
-                    panic!("expected symbol");
+                self.error("Unknown character encountered");
-                }
+                TokenKind::Invalid
-            },
-            None => {
-                self.error();
-                panic!("expected symbol");
             }
-        }
+        };
+        self.new_token(typ)
     }
 
-    fn get_alphanumeric(&mut self) -> Token<'a> {
+    fn lex(&mut self) {
-        while let Some(c) = self.chars.peek() {
+        self.skip_whitespace();
+        self.start = self.end;
+
+        let token = if let Some(c) = self.peek() {
             match c {
-                'a'..='z' | 'A'..='Z' | '0'..='9' => {}
-                _ => break,
-            }
-            self.next();
-        }
-
-        use Token::Keyword;
-        use TokenKeyword::*;
-
-        match &self.text[self.start..self.end] {
-            "let" => Keyword(Let),
-            "fn" => Keyword(Fn),
-            "ret" => Keyword(Ret),
-            "if" => Keyword(If),
-            "else" => Keyword(Else),
-            "elif" => Keyword(Elif),
-            "while" => Keyword(While),
-            "do" => Keyword(Do),
-            "for" => Keyword(For),
-            "int" => Keyword(Int),
-            "float" => Keyword(Float),
-            "char" => Keyword(Char),
-            _ => Token::Identifier(&self.text[self.start..self.end]),
-        }
-    }
-
-    pub fn lex(&mut self) -> Vec<Token<'a>> {
-        let mut tokens: Vec<Token> = Vec::new();
-
-        while let Some(c) = self.chars.peek() {
-            match c {
-                ' ' | '\r' | '\t' => {
-                    self.next();
-                }
-                '\\' => {
-                    self.next();
-                    self.escape_newline();
-                }
                 '\n' => {
-                    tokens.push(Token::Newline);
-                    self.next();
                     self.line += 1;
+                    self.new_token(TokenKind::Newline)
                 }
-                '0'..='9' => tokens.push(self.get_numeric()),
+                '0'..='9' => self.get_numeric(),
-                '\'' => tokens.push(self.get_char()),
+                'a'..='z' | 'A'..='Z' => self.get_alphanumeric(),
-                '{' | '}' | '(' | ')' => tokens.push(self.get_delimiter()),
+                '\'' => self.get_char(),
-                '+' | '-' | '*' | '/' | '%' | '^' | '~' | '&' | '|' | '!' | '<' | '>' | '='
+                _ => self.get_symbol(),
-                | ':' | '.' | '#' => tokens.push(self.get_symbol()),
-                'a'..='z' | 'A'..='Z' => tokens.push(self.get_alphanumeric()),
-                _ => {
-                    self.error();
-                    panic!("unknown character encountered");
-                }
             }
+        } else {
+            self.new_token(TokenKind::Eof)
+        };
+        self.tokens.push_back(token);
+    }
 
-            self.start = self.end;
+    /// Peeks at the next token and returns a reference to it
+    pub fn peek_token(&mut self) -> &Token {
+        if self.tokens.is_empty() {
+            self.lex();
         }
+        &self.tokens[0]
+    }
 
-        tokens
+    /// Returns the next token, moving the lexer forward
+    pub fn next_token(&mut self) -> Token {
+        if self.tokens.is_empty() {
+            self.lex();
+        }
+        self.tokens.pop_front().unwrap()
     }
 }
+
+#[test]
+fn test_peek_next() {
+    let mut lexer = Lexer::new("test01");
+    assert_eq!(lexer.peek(), Some(&'t'));
+    assert_eq!(lexer.next(), Some('t'));
+    assert_eq!(lexer.peek(), Some(&'e'));
+    assert_eq!(lexer.peek(), Some(&'e'));
+    assert_eq!(lexer.next(), Some('e'));
+    assert_eq!(lexer.next(), Some('s'));
+    assert_eq!(lexer.next(), Some('t'));
+    assert_eq!(lexer.next(), Some('0'));
+    assert_eq!(lexer.peek(), Some(&'1'));
+    assert_eq!(lexer.next(), Some('1'));
+    assert_eq!(lexer.peek(), None);
+    assert_eq!(lexer.next(), None);
+    assert_eq!(lexer.peek(), None);
+}
+
+#[test]
+fn test_tokens_1() {
+    let mut lexer = Lexer::new("let test02 = 4 << 1");
+
+    use TokenKind::*;
+
+    assert_eq!(lexer.peek_token().kind, Keyword(TokenKeyword::Let));
+    assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Let));
+
+    let mut token = lexer.next_token();
+    assert_eq!(token.kind, Identifier);
+    assert_eq!(*token.val, *"test02");
+
+    assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Eq));
+
+    token = lexer.next_token();
+    assert_eq!(token.kind, Literal(TokenLiteral::Int));
+    assert_eq!(*token.val, *"4");
+
+    assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Shl));
+
+    assert_eq!(lexer.peek_token().kind, Literal(TokenLiteral::Int));
+    assert_eq!(*lexer.peek_token().val, *"1");
+    token = lexer.next_token();
+    assert_eq!(token.kind, Literal(TokenLiteral::Int));
+    assert_eq!(*token.val, *"1");
+
+    assert_eq!(lexer.peek_token().kind, Eof);
+    assert_eq!(lexer.next_token().kind, Eof);
+    assert_eq!(lexer.peek_token().kind, Eof);
+    assert_eq!(lexer.next_token().kind, Eof);
+}
+
+#[test]
+fn test_tokens_2() {
+    let mut lexer = Lexer::new("let test03: char = 'h'");
+
+    use TokenKind::*;
+
+    assert_eq!(lexer.peek_token().kind, Keyword(TokenKeyword::Let));
+    assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Let));
+
+    let mut token = lexer.next_token();
+    assert_eq!(token.kind, Identifier);
+    assert_eq!(*token.val, *"test03");
+
+    assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Colon));
+    assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Char));
+    assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Eq));
+
+    assert_eq!(lexer.peek_token().kind, Literal(TokenLiteral::Char));
+    assert_eq!(*lexer.peek_token().val, *"'h'");
+    token = lexer.next_token();
+    assert_eq!(token.kind, Literal(TokenLiteral::Char));
+    assert_eq!(*token.val, *"'h'");
+
+    assert_eq!(lexer.peek_token().kind, Eof);
+    assert_eq!(lexer.next_token().kind, Eof);
+}
+
+#[test]
+fn test_tokens_3() {
+    let mut lexer = Lexer::new("");
+
+    assert_eq!(lexer.peek_token().kind, TokenKind::Eof);
+    assert_eq!(lexer.next_token().kind, TokenKind::Eof);
+}

src/main.rs

@@ -1,8 +1,13 @@
-use std::fs;
+use std::{
-use std::panic;
+    fs,
+    panic,
+};
 
 use tricc::args::Args;
-use tricc::lexer::Lexer;
+use tricc::lexer::{
+    Lexer,
+    TokenKind,
+};
 
 fn main() {
     panic::set_hook(Box::new(|panic_info| {
@@ -21,14 +26,15 @@ fn main() {
         }
     }));
 
-    let mut args = Args::new();
+    let mut args = Args::default();
     args.handle();
 
     let file = args.get_file();
-    let contents = fs::read_to_string(&file).expect("Couldn't read the file");
+    let content = fs::read_to_string(&file).expect("Couldn't read the file");
 
-    let mut lexer = Lexer::new(&file, contents.as_str());
+    let mut lexer = Lexer::new(content.as_str());
-    let tokens = lexer.lex();
 
-    println!("{:?}", tokens);
+    while lexer.peek_token().kind != TokenKind::Eof {
+        println!("{:?}", lexer.next_token());
+    }
 }