diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a5e057d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI + +on: [push, pull_request, workflow_dispatch] + +jobs: + checks: + name: Checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v20 + with: + extra_nix_config: | + auto-optimise-store = true + experimental-features = nix-command flakes + - uses: cachix/cachix-action@v12 + with: + name: pain + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - name: fmt check + run: nix build .#checks.fmt -L + + - name: clippy check + run: nix build .#checks.clippy -L + + - name: nextest check + run: nix build .#checks.nextest -L + + - name: doc tests + run: nix build .#checks.doc -L + + build: + name: Build + runs-on: ubuntu-latest + needs: [ checks ] + steps: + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v20 + with: + extra_nix_config: | + auto-optimise-store = true + experimental-features = nix-command flakes + - uses: cachix/cachix-action@v12 + with: + name: pain + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - name: fmt check + run: nix build .#tricc -L \ No newline at end of file diff --git a/flake.lock b/flake.lock index 0192d55..01bf2fb 100644 --- a/flake.lock +++ b/flake.lock @@ -10,11 +10,11 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1680584903, - "narHash": "sha256-uraq+D3jcLzw/UVk0xMHcnfILfIMa0DLrtAEq2nNlxU=", + "lastModified": 1688772518, + "narHash": "sha256-ol7gZxwvgLnxNSZwFTDJJ49xVY5teaSvF7lzlo3YQfM=", "owner": "ipetkov", "repo": "crane", - "rev": "65d3f6a3970cd46bef5eedfd458300f72c56b3c5", + "rev": "8b08e96c9af8c6e3a2b69af5a7fa168750fcf88e", "type": "github" }, "original": { @@ -40,12 +40,15 @@ } }, "flake-utils": { + "inputs": { + "systems": "systems" + }, "locked": { - "lastModified": 1678901627, - "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", + "lastModified": 1687709756, + "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", "owner": "numtide", "repo": "flake-utils", - "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", + "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", "type": "github" }, "original": { @@ -55,12 +58,15 @@ } }, "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, "locked": { - "lastModified": 1678901627, - "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", "owner": "numtide", "repo": "flake-utils", - "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", "type": "github" }, "original": { @@ -70,12 +76,15 @@ } }, "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", "owner": "numtide", "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", "type": "github" }, "original": { @@ -86,11 +95,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1680724564, - "narHash": "sha256-eeUUGOTKTelYKDbUxKs0V7GUa186L2fym7jM2QQ4Oss=", + "lastModified": 1688392541, + "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "36adaa6aaa6b03e59102df0c1b12cdc3f23fd112", + "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", "type": "github" }, "original": { @@ -120,11 +129,11 @@ ] }, "locked": { - "lastModified": 1680488274, - "narHash": "sha256-0vYMrZDdokVmPQQXtFpnqA2wEgCCUXf5a3dDuDVshn0=", + "lastModified": 1688351637, + "narHash": "sha256-CLTufJ29VxNOIZ8UTg0lepsn3X03AmopmaLTTeHDCL4=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "7ec2ff598a172c6e8584457167575b3a1a5d80d8", + "rev": "f9b92316727af9e6c7fee4a761242f7f46880329", "type": "github" }, "original": { @@ -141,11 +150,11 @@ ] }, "locked": { - "lastModified": 1680660688, - "narHash": "sha256-XeQTCxWBR0Ai1VMzI5ZXYpA2lu1F8FzZKjw8RtByZOg=", + "lastModified": 1690252178, + "narHash": "sha256-9oEz822bvbHobfCUjJLDor2BqW3I5tycIauzDlzOALY=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "2f40052be98347b479c820c00fb2fc1d87b3aa28", + "rev": "8d64353ca827002fb8459e44d49116c78d868eba", "type": "github" }, "original": { @@ -153,6 +162,51 @@ "repo": "rust-overlay", "type": "github" } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index b1b857f..0cd34ad 100644 --- a/flake.nix +++ b/flake.nix @@ -38,25 +38,34 @@ tricc = craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; + doCheck = false; }); in { - checks = { - inherit tricc; - - clippy = craneLib.cargoClippy (commonArgs // { - inherit cargoArtifacts; - cargoClippyExtraArgs = "--all-targets -- --deny warnings"; - }); - - fmt = craneLib.cargoFmt { - inherit src; - }; - }; - packages = { inherit tricc; default = tricc; + + # not using flake checks to run them individually + checks = { + clippy = craneLib.cargoClippy (commonArgs // { + inherit cargoArtifacts; + }); + + fmt = craneLib.cargoFmt { + inherit src; + }; + + doc = craneLib.cargoDoc (commonArgs // { + inherit cargoArtifacts; + }); + + nextest = craneLib.cargoNextest (commonArgs // { + inherit cargoArtifacts; + partitions = 1; + partitionType = "count"; + }); + }; }; devShells.default = pkgs.mkShell { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index a713b91..8e68c42 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-04-01" +channel = "nightly-2023-07-15" components = [ "rustfmt", "clippy", "rust-analyzer", "rust-src" ] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..3ddb571 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,5 @@ +comment_width = 99 +format_code_in_doc_comments = true +imports_granularity = "Module" +imports_layout = "Vertical" +wrap_comments = true \ No newline at end of file diff --git a/src/args.rs b/src/args.rs index e1dab96..17d042d 100644 --- a/src/args.rs +++ b/src/args.rs @@ -4,7 +4,7 @@ use std::process::exit; const VERSION: &str = env!("CARGO_PKG_VERSION"); const CRATE: &str = env!("CARGO_CRATE_NAME"); -// naive argument handling +/// A naive argument handler #[derive(Default)] pub struct Args { version: bool, @@ -12,13 +12,12 @@ pub struct Args { } impl Args { + /// Creates a new [`Args`] instance pub fn new() -> Args { - Args { - version: false, - file: None, - } + Args::default() } + /// Checks for various arguments pub fn handle(&mut self) { let args: Vec = env::args().collect(); @@ -53,6 +52,8 @@ impl Args { } } + /// Fetches the file from the arguments. + /// Panics if there is no file in the arguments #[inline] pub fn get_file(self) -> String { self.file.expect("no file supplied!") diff --git a/src/lexer.rs b/src/lexer.rs index 4fd47c4..924b963 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,13 +1,22 @@ -use std::{iter, str}; +use std::collections::VecDeque; +use std::iter::Peekable; +use std::rc::Rc; +use std::str; -#[derive(Debug)] +/// All token literals +/// +/// TODO: Add string +#[derive(Debug, PartialEq)] pub enum TokenLiteral { Int, Float, Char, } -#[derive(Debug)] +/// All token symbols +/// +/// TODO: Maybe add * +#[derive(Debug, PartialEq)] pub enum TokenSymbol { // operators Plus, @@ -58,10 +67,16 @@ pub enum TokenSymbol { Hash, } -#[derive(Debug)] +/// All token keywod +#[derive(Debug, PartialEq)] pub enum TokenKeyword { - Let, + // parents Fn, + Class, + Module, + + // statements + Let, Ret, // conditionals @@ -69,10 +84,10 @@ pub enum TokenKeyword { Else, Elif, - // loops - While, - Do, - For, + // control flow + Loop, + Break, + Continue, // primitives Int, @@ -80,7 +95,10 @@ pub enum TokenKeyword { Char, } -#[derive(Debug)] +/// All token delimiters +/// +/// TODO: Maybe add \[ and \] +#[derive(Debug, PartialEq)] pub enum TokenDelimiter { BraceOpen, BraceClose, @@ -88,31 +106,70 @@ pub enum TokenDelimiter { ParenClose, } -#[derive(Debug)] -pub enum Token<'a> { +/// All tokens +#[derive(Debug, PartialEq)] +pub enum TokenKind { Newline, - Literal(TokenLiteral, &'a str), + Eof, + Literal(TokenLiteral), Symbol(TokenSymbol), Keyword(TokenKeyword), Delimiter(TokenDelimiter), - Identifier(&'a str), + Identifier, + Invalid, +} + +#[derive(Debug)] +pub struct Token { + pub kind: TokenKind, + /// Holds the reference to the tokenized string + /// + /// For example, if `kind` is of type [`TokenKind::Identifier`], this would contain the value + /// of that identifier + pub val: Rc, } pub struct Lexer<'a> { - file: &'a str, + /// The entire text to be tokenized text: &'a str, - chars: iter::Peekable>, - line: usize, - start: usize, + /// A peekable iterate for `text` + chars: Peekable>, + /// A peekable double ended queue for the tokens + tokens: VecDeque, + /// Current line number + pub line: usize, + /// Start character index for the current token + pub start: usize, + /// End character index for the current token end: usize, } impl<'a> Lexer<'a> { - pub fn new(file: &'a str, contents: &'a str) -> Lexer<'a> { + /// Creates a new [`Lexer`] instance with the provided content. + /// + /// The `Lexer` is responsible for tokenizing the given text, making it easier to + /// perform various parsing operations. + /// + /// # Arguments + /// + /// * `content`: The text to tokenize. + /// + /// # Returns + /// + /// A new instance of `Lexer` initialized with the provided `content`. + /// + /// # Example + /// + /// ``` + /// use tricc::lexer::Lexer; + /// + /// let lexer = Lexer::new("let example: int = 4"); + /// ``` + pub fn new(content: &'a str) -> Self { Lexer { - file, - text: contents, - chars: contents.chars().peekable(), + text: content, + chars: content.chars().peekable(), + tokens: VecDeque::new(), line: 1, start: 0, end: 0, @@ -120,49 +177,63 @@ impl<'a> Lexer<'a> { } #[inline] - fn error(&self) { - eprintln!("error lexing \"{}:{}:{}\"", self.file, self.line, self.end); + fn new_token(&self, kind: TokenKind) -> Token { + Token { + kind, + val: Rc::from(&self.text[self.start..self.end]), + } } + #[inline] + fn error(&self, msg: &str) { + eprintln!("Lexer: {}, at \"{}:{}\"", msg, self.line, self.end); + } + + #[inline] + fn peek(&mut self) -> Option<&char> { + self.chars.peek() + } + + #[inline] fn next(&mut self) -> Option { self.end += 1; self.chars.next() } - fn skip(&mut self, c: char) { - if self.next() != Some(c) { - self.error(); - panic!("expected {}", c); - } - } + fn skip_whitespace(&mut self) { + let mut ignore_nl: bool = false; - fn escape_newline(&mut self) { - while let Some(c) = self.chars.peek() { + while let Some(c) = self.peek() { match c { '\r' | '\t' | ' ' => { self.next(); } '\n' => { - self.next(); - break; + if ignore_nl { + ignore_nl = false; + self.next(); + } else { + break; + } } - _ => { - self.error(); - panic!("expected newline"); - }, + '\\' => { + self.next(); + ignore_nl = true; + } + _ => break, } } } - fn get_numeric(&mut self) -> Token<'a> { + fn get_numeric(&mut self) -> Token { let mut is_float: bool = false; - while let Some(c) = self.chars.peek() { + while let Some(c) = self.peek() { match c { '0'..='9' => {} '.' => { if is_float { - self.error(); - panic!("multiple decimals encountered") + self.error("Multiple decimals encountered"); + return self.new_token(TokenKind::Invalid); } is_float = true; } @@ -171,62 +242,78 @@ impl<'a> Lexer<'a> { self.next(); } - Token::Literal( - if is_float { - TokenLiteral::Float - } else { - TokenLiteral::Int - }, - &self.text[self.start..self.end], - ) + self.new_token(TokenKind::Literal(if is_float { + TokenLiteral::Float + } else { + TokenLiteral::Int + })) } - fn get_char(&mut self) -> Token<'a> { - self.skip('\''); + fn get_char(&mut self) -> Token { + // skip ' + self.next(); if matches!(self.next(), Some('\'') | None) { - self.error(); - panic!("A character literal cannot be empty"); + self.error("Expected character literal"); + return self.new_token(TokenKind::Invalid); } - self.skip('\''); + // skip ' + self.next(); - Token::Literal(TokenLiteral::Char, &self.text[self.start + 1..self.end - 1]) + self.new_token(TokenKind::Literal(TokenLiteral::Char)) } - fn get_delimiter(&mut self) -> Token<'a> { - use Token::Delimiter; - use TokenDelimiter::*; - - match self.next() { - Some(c) => match c { - '{' => Delimiter(BraceOpen), - '}' => Delimiter(BraceClose), - '(' => Delimiter(ParenOpen), - ')' => Delimiter(ParenClose), - _ => { - self.error(); - panic!("expected delimiter"); - } - }, - None => { - self.error(); - panic!("expected delimiter"); + fn get_alphanumeric(&mut self) -> Token { + while let Some(c) = self.peek() { + match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' => {} + _ => break, } + self.next(); } + + use TokenKeyword::*; + use TokenKind::Keyword; + + self.new_token(match &self.text[self.start..self.end] { + "fn" => Keyword(Fn), + "class" => Keyword(Class), + "module" => Keyword(Module), + "let" => Keyword(Let), + "ret" => Keyword(Ret), + "if" => Keyword(If), + "else" => Keyword(Else), + "elif" => Keyword(Elif), + "loop" => Keyword(Loop), + "break" => Keyword(Break), + "continue" => Keyword(Continue), + "int" => Keyword(Int), + "float" => Keyword(Float), + "char" => Keyword(Char), + _ => TokenKind::Identifier, + }) } - fn get_symbol(&mut self) -> Token<'a> { - use Token::Symbol; + fn get_symbol(&mut self) -> Token { + let c = self.next().unwrap(); + + use TokenDelimiter::*; + use TokenKind::{ + Delimiter, + Symbol, + }; + use TokenSymbol::*; // handle +, +=, -, -=, *, *=, /, /=, %, %=, ^, ^=, !, != macro_rules! token_symbol_eq { ($a:expr, $b:expr) => { - if self.chars.peek() == Some(&'=') { - self.next(); - Symbol($b) - } else { - Symbol($a) + match self.peek() { + Some('=') => { + self.next(); + Symbol($b) + } + _ => Symbol($a), } }; } @@ -234,7 +321,7 @@ impl<'a> Lexer<'a> { // handle &, |, ||, &&, &=, |= macro_rules! token_symbol_logical { ($a:expr, $b:expr, $c:expr, $d:expr) => { - match self.chars.peek() { + match self.peek() { Some('=') => { self.next(); Symbol($c) @@ -251,7 +338,7 @@ impl<'a> Lexer<'a> { // handle <, <=, >, >=, <<, >>, <<=, >>= macro_rules! token_symbol_compare { ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => { - match self.chars.peek() { + match self.peek() { Some('=') => { self.next(); Symbol($d) @@ -265,99 +352,155 @@ impl<'a> Lexer<'a> { }; } - use TokenSymbol::*; - - match self.next() { - Some(c) => match c { - '+' => token_symbol_eq!(Plus, PlusEq), - '-' => token_symbol_eq!(Minus, MinusEq), - '*' => token_symbol_eq!(Star, StarEq), - '/' => token_symbol_eq!(Slash, SlashEq), - '%' => token_symbol_eq!(Percent, PercentEq), - '^' => token_symbol_eq!(Caret, CaretEq), - '!' => token_symbol_eq!(Not, Ne), - '=' => token_symbol_eq!(Eq, EqEq), - '&' => token_symbol_logical!(And, AndAnd, AndEq, '&'), - '|' => token_symbol_logical!(Or, OrOr, OrEq, '|'), - '<' => token_symbol_compare!(Lt, Shl, ShlEq, LtEq, '<'), - '>' => token_symbol_compare!(Gt, Shr, ShrEq, GtEq, '>'), - '~' => Symbol(Tilde), - ':' => Symbol(Colon), - '.' => Symbol(Dot), - '#' => Symbol(Hash), - _ => { - self.error(); - panic!("expected symbol"); - } - }, - None => { - self.error(); - panic!("expected symbol"); + let typ = match c { + '{' => Delimiter(BraceOpen), + '}' => Delimiter(BraceClose), + '(' => Delimiter(ParenOpen), + ')' => Delimiter(ParenClose), + '+' => token_symbol_eq!(Plus, PlusEq), + '-' => token_symbol_eq!(Minus, MinusEq), + '*' => token_symbol_eq!(Star, StarEq), + '/' => token_symbol_eq!(Slash, SlashEq), + '%' => token_symbol_eq!(Percent, PercentEq), + '^' => token_symbol_eq!(Caret, CaretEq), + '!' => token_symbol_eq!(Not, Ne), + '=' => token_symbol_eq!(Eq, EqEq), + '&' => token_symbol_logical!(And, AndAnd, AndEq, '&'), + '|' => token_symbol_logical!(Or, OrOr, OrEq, '|'), + '<' => token_symbol_compare!(Lt, Shl, ShlEq, LtEq, '<'), + '>' => token_symbol_compare!(Gt, Shr, ShrEq, GtEq, '>'), + '~' => Symbol(Tilde), + ':' => Symbol(Colon), + '.' => Symbol(Dot), + '#' => Symbol(Hash), + _ => { + self.error("Unknown character encountered"); + TokenKind::Invalid } - } + }; + self.new_token(typ) } - fn get_alphanumeric(&mut self) -> Token<'a> { - while let Some(c) = self.chars.peek() { + fn lex(&mut self) { + self.skip_whitespace(); + self.start = self.end; + + let token = if let Some(c) = self.peek() { match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' => {} - _ => break, - } - self.next(); - } - - use Token::Keyword; - use TokenKeyword::*; - - match &self.text[self.start..self.end] { - "let" => Keyword(Let), - "fn" => Keyword(Fn), - "ret" => Keyword(Ret), - "if" => Keyword(If), - "else" => Keyword(Else), - "elif" => Keyword(Elif), - "while" => Keyword(While), - "do" => Keyword(Do), - "for" => Keyword(For), - "int" => Keyword(Int), - "float" => Keyword(Float), - "char" => Keyword(Char), - _ => Token::Identifier(&self.text[self.start..self.end]), - } - } - - pub fn lex(&mut self) -> Vec> { - let mut tokens: Vec = Vec::new(); - - while let Some(c) = self.chars.peek() { - match c { - ' ' | '\r' | '\t' => { - self.next(); - } - '\\' => { - self.next(); - self.escape_newline(); - } '\n' => { - tokens.push(Token::Newline); - self.next(); self.line += 1; + self.new_token(TokenKind::Newline) } - '0'..='9' => tokens.push(self.get_numeric()), - '\'' => tokens.push(self.get_char()), - '{' | '}' | '(' | ')' => tokens.push(self.get_delimiter()), - '+' | '-' | '*' | '/' | '%' | '^' | '~' | '&' | '|' | '!' | '<' | '>' | '=' - | ':' | '.' | '#' => tokens.push(self.get_symbol()), - 'a'..='z' | 'A'..='Z' => tokens.push(self.get_alphanumeric()), - _ => { - self.error(); - panic!("unknown character encountered"); - } + '0'..='9' => self.get_numeric(), + 'a'..='z' | 'A'..='Z' => self.get_alphanumeric(), + '\'' => self.get_char(), + _ => self.get_symbol(), } + } else { + self.new_token(TokenKind::Eof) + }; + self.tokens.push_back(token); + } - self.start = self.end; + /// Peeks at the next token and returns a reference to it + pub fn peek_token(&mut self) -> &Token { + if self.tokens.is_empty() { + self.lex(); } + &self.tokens[0] + } - tokens + /// Returns the next token, moving the lexer forward + pub fn next_token(&mut self) -> Token { + if self.tokens.is_empty() { + self.lex(); + } + self.tokens.pop_front().unwrap() } } + +#[test] +fn test_peek_next() { + let mut lexer = Lexer::new("test01"); + assert_eq!(lexer.peek(), Some(&'t')); + assert_eq!(lexer.next(), Some('t')); + assert_eq!(lexer.peek(), Some(&'e')); + assert_eq!(lexer.peek(), Some(&'e')); + assert_eq!(lexer.next(), Some('e')); + assert_eq!(lexer.next(), Some('s')); + assert_eq!(lexer.next(), Some('t')); + assert_eq!(lexer.next(), Some('0')); + assert_eq!(lexer.peek(), Some(&'1')); + assert_eq!(lexer.next(), Some('1')); + assert_eq!(lexer.peek(), None); + assert_eq!(lexer.next(), None); + assert_eq!(lexer.peek(), None); +} + +#[test] +fn test_tokens_1() { + let mut lexer = Lexer::new("let test02 = 4 << 1"); + + use TokenKind::*; + + assert_eq!(lexer.peek_token().kind, Keyword(TokenKeyword::Let)); + assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Let)); + + let mut token = lexer.next_token(); + assert_eq!(token.kind, Identifier); + assert_eq!(*token.val, *"test02"); + + assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Eq)); + + token = lexer.next_token(); + assert_eq!(token.kind, Literal(TokenLiteral::Int)); + assert_eq!(*token.val, *"4"); + + assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Shl)); + + assert_eq!(lexer.peek_token().kind, Literal(TokenLiteral::Int)); + assert_eq!(*lexer.peek_token().val, *"1"); + token = lexer.next_token(); + assert_eq!(token.kind, Literal(TokenLiteral::Int)); + assert_eq!(*token.val, *"1"); + + assert_eq!(lexer.peek_token().kind, Eof); + assert_eq!(lexer.next_token().kind, Eof); + assert_eq!(lexer.peek_token().kind, Eof); + assert_eq!(lexer.next_token().kind, Eof); +} + +#[test] +fn test_tokens_2() { + let mut lexer = Lexer::new("let test03: char = 'h'"); + + use TokenKind::*; + + assert_eq!(lexer.peek_token().kind, Keyword(TokenKeyword::Let)); + assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Let)); + + let mut token = lexer.next_token(); + assert_eq!(token.kind, Identifier); + assert_eq!(*token.val, *"test03"); + + assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Colon)); + assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Char)); + assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Eq)); + + assert_eq!(lexer.peek_token().kind, Literal(TokenLiteral::Char)); + assert_eq!(*lexer.peek_token().val, *"'h'"); + token = lexer.next_token(); + assert_eq!(token.kind, Literal(TokenLiteral::Char)); + assert_eq!(*token.val, *"'h'"); + + assert_eq!(lexer.peek_token().kind, Eof); + assert_eq!(lexer.next_token().kind, Eof); +} + +#[test] +fn test_tokens_3() { + let mut lexer = Lexer::new(""); + + assert_eq!(lexer.peek_token().kind, TokenKind::Eof); + assert_eq!(lexer.next_token().kind, TokenKind::Eof); +} diff --git a/src/main.rs b/src/main.rs index 5b3ce2a..c14239c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,13 @@ -use std::fs; -use std::panic; +use std::{ + fs, + panic, +}; use tricc::args::Args; -use tricc::lexer::Lexer; +use tricc::lexer::{ + Lexer, + TokenKind, +}; fn main() { panic::set_hook(Box::new(|panic_info| { @@ -21,14 +26,15 @@ fn main() { } })); - let mut args = Args::new(); + let mut args = Args::default(); args.handle(); let file = args.get_file(); - let contents = fs::read_to_string(&file).expect("Couldn't read the file"); + let content = fs::read_to_string(&file).expect("Couldn't read the file"); - let mut lexer = Lexer::new(&file, contents.as_str()); - let tokens = lexer.lex(); + let mut lexer = Lexer::new(content.as_str()); - println!("{:?}", tokens); + while lexer.peek_token().kind != TokenKind::Eof { + println!("{:?}", lexer.next_token()); + } }