natto1784/tricc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

revive project

Browse Source 
Signed-off-by: Amneesh Singh <natto@weirdnatto.in>
This commit is contained in:
Amneesh Singh
2023-07-29 20:28:14 +05:30
parent 879d3d3b65
commit e8192df9e2
8 changed files with 482 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,50 @@
name: CI
on: [push, pull_request, workflow_dispatch]
jobs:
checks:
name: Checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v20
with:
extra_nix_config: |
auto-optimise-store = true
experimental-features = nix-command flakes
- uses: cachix/cachix-action@v12
with:
name: pain
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- name: fmt check
run: nix build .#checks.fmt -L
- name: clippy check
run: nix build .#checks.clippy -L
- name: nextest check
run: nix build .#checks.nextest -L
- name: doc tests
run: nix build .#checks.doc -L
build:
name: Build
runs-on: ubuntu-latest
needs: [ checks ]
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v20
with:
extra_nix_config: |
auto-optimise-store = true
experimental-features = nix-command flakes
- uses: cachix/cachix-action@v12
with:
name: pain
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- name: fmt check
run: nix build .#tricc -L

96
flake.lock generated
View File

@@ -10,11 +10,11 @@
"rust-overlay": "rust-overlay"
},
"locked": {
"lastModified": 1680584903,
"narHash": "sha256-uraq+D3jcLzw/UVk0xMHcnfILfIMa0DLrtAEq2nNlxU=",
"lastModified": 1688772518,
"narHash": "sha256-ol7gZxwvgLnxNSZwFTDJJ49xVY5teaSvF7lzlo3YQfM=",
"owner": "ipetkov",
"repo": "crane",
"rev": "65d3f6a3970cd46bef5eedfd458300f72c56b3c5",
"rev": "8b08e96c9af8c6e3a2b69af5a7fa168750fcf88e",
"type": "github"
},
"original": {
@@ -40,12 +40,15 @@
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1678901627,
"narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
"lastModified": 1687709756,
"narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
"rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
"type": "github"
},
"original": {
@@ -55,12 +58,15 @@
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1678901627,
"narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
"lastModified": 1689068808,
"narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
"rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
"type": "github"
},
"original": {
@@ -70,12 +76,15 @@
}
},
"flake-utils_3": {
"inputs": {
"systems": "systems_3"
},
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
@@ -86,11 +95,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1680724564,
"narHash": "sha256-eeUUGOTKTelYKDbUxKs0V7GUa186L2fym7jM2QQ4Oss=",
"lastModified": 1688392541,
"narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "36adaa6aaa6b03e59102df0c1b12cdc3f23fd112",
"rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
"type": "github"
},
"original": {
@@ -120,11 +129,11 @@
]
},
"locked": {
"lastModified": 1680488274,
"narHash": "sha256-0vYMrZDdokVmPQQXtFpnqA2wEgCCUXf5a3dDuDVshn0=",
"lastModified": 1688351637,
"narHash": "sha256-CLTufJ29VxNOIZ8UTg0lepsn3X03AmopmaLTTeHDCL4=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "7ec2ff598a172c6e8584457167575b3a1a5d80d8",
"rev": "f9b92316727af9e6c7fee4a761242f7f46880329",
"type": "github"
},
"original": {
@@ -141,11 +150,11 @@
]
},
"locked": {
"lastModified": 1680660688,
"narHash": "sha256-XeQTCxWBR0Ai1VMzI5ZXYpA2lu1F8FzZKjw8RtByZOg=",
"lastModified": 1690252178,
"narHash": "sha256-9oEz822bvbHobfCUjJLDor2BqW3I5tycIauzDlzOALY=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "2f40052be98347b479c820c00fb2fc1d87b3aa28",
"rev": "8d64353ca827002fb8459e44d49116c78d868eba",
"type": "github"
},
"original": {
@@ -153,6 +162,51 @@
"repo": "rust-overlay",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_3": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",

35
flake.nix
View File

@@ -38,25 +38,34 @@
tricc = craneLib.buildPackage (commonArgs // {
inherit cargoArtifacts;
doCheck = false;
});
in
{
checks = {
inherit tricc;
clippy = craneLib.cargoClippy (commonArgs // {
inherit cargoArtifacts;
cargoClippyExtraArgs = "--all-targets -- --deny warnings";
});
fmt = craneLib.cargoFmt {
inherit src;
};
};
packages = {
inherit tricc;
default = tricc;
# not using flake checks to run them individually
checks = {
clippy = craneLib.cargoClippy (commonArgs // {
inherit cargoArtifacts;
});
fmt = craneLib.cargoFmt {
inherit src;
};
doc = craneLib.cargoDoc (commonArgs // {
inherit cargoArtifacts;
});
nextest = craneLib.cargoNextest (commonArgs // {
inherit cargoArtifacts;
partitions = 1;
partitionType = "count";
});
};
};
devShells.default = pkgs.mkShell {

2
rust-toolchain.toml
View File

@@ -1,3 +1,3 @@
[toolchain]
channel = "nightly-2023-04-01"
channel = "nightly-2023-07-15"
components = [ "rustfmt", "clippy", "rust-analyzer", "rust-src" ]

5
rustfmt.toml Normal file
View File

@@ -0,0 +1,5 @@
comment_width = 99
format_code_in_doc_comments = true
imports_granularity = "Module"
imports_layout = "Vertical"
wrap_comments = true

11
src/args.rs
View File

@@ -4,7 +4,7 @@ use std::process::exit;
const VERSION: &str = env!("CARGO_PKG_VERSION");
const CRATE: &str = env!("CARGO_CRATE_NAME");
// naive argument handling
/// A naive argument handler
#[derive(Default)]
pub struct Args {
version: bool,
@@ -12,13 +12,12 @@ pub struct Args {
}
impl Args {
/// Creates a new [`Args`] instance
pub fn new() -> Args {
Args {
version: false,
file: None,
}
Args::default()
}
/// Checks for various arguments
pub fn handle(&mut self) {
let args: Vec<String> = env::args().collect();
@@ -53,6 +52,8 @@ impl Args {
}
}
/// Fetches the file from the arguments.
/// Panics if there is no file in the arguments
#[inline]
pub fn get_file(self) -> String {
self.file.expect("no file supplied!")

475
src/lexer.rs
View File

@@ -1,13 +1,22 @@
use std::{iter, str};
use std::collections::VecDeque;
use std::iter::Peekable;
use std::rc::Rc;
use std::str;
#[derive(Debug)]
/// All token literals
///
/// TODO: Add string
#[derive(Debug, PartialEq)]
pub enum TokenLiteral {
Int,
Float,
Char,
}
#[derive(Debug)]
/// All token symbols
///
/// TODO: Maybe add *
#[derive(Debug, PartialEq)]
pub enum TokenSymbol {
// operators
Plus,
@@ -58,10 +67,16 @@ pub enum TokenSymbol {
Hash,
}
#[derive(Debug)]
/// All token keywod
#[derive(Debug, PartialEq)]
pub enum TokenKeyword {
Let,
// parents
Fn,
Class,
Module,
// statements
Let,
Ret,
// conditionals
@@ -69,10 +84,10 @@ pub enum TokenKeyword {
Else,
Elif,
// loops
While,
Do,
For,
// control flow
Loop,
Break,
Continue,
// primitives
Int,
@@ -80,7 +95,10 @@ pub enum TokenKeyword {
Char,
}
#[derive(Debug)]
/// All token delimiters
///
/// TODO: Maybe add \[ and \]
#[derive(Debug, PartialEq)]
pub enum TokenDelimiter {
BraceOpen,
BraceClose,
@@ -88,31 +106,70 @@ pub enum TokenDelimiter {
ParenClose,
}
#[derive(Debug)]
pub enum Token<'a> {
/// All tokens
#[derive(Debug, PartialEq)]
pub enum TokenKind {
Newline,
Literal(TokenLiteral, &'a str),
Eof,
Literal(TokenLiteral),
Symbol(TokenSymbol),
Keyword(TokenKeyword),
Delimiter(TokenDelimiter),
Identifier(&'a str),
Identifier,
Invalid,
}
#[derive(Debug)]
pub struct Token {
pub kind: TokenKind,
/// Holds the reference to the tokenized string
///
/// For example, if `kind` is of type [`TokenKind::Identifier`], this would contain the value
/// of that identifier
pub val: Rc<str>,
}
pub struct Lexer<'a> {
file: &'a str,
/// The entire text to be tokenized
text: &'a str,
chars: iter::Peekable<str::Chars<'a>>,
line: usize,
start: usize,
/// A peekable iterate for `text`
chars: Peekable<str::Chars<'a>>,
/// A peekable double ended queue for the tokens
tokens: VecDeque<Token>,
/// Current line number
pub line: usize,
/// Start character index for the current token
pub start: usize,
/// End character index for the current token
end: usize,
}
impl<'a> Lexer<'a> {
pub fn new(file: &'a str, contents: &'a str) -> Lexer<'a> {
/// Creates a new [`Lexer`] instance with the provided content.
///
/// The `Lexer` is responsible for tokenizing the given text, making it easier to
/// perform various parsing operations.
///
/// # Arguments
///
/// * `content`: The text to tokenize.
///
/// # Returns
///
/// A new instance of `Lexer` initialized with the provided `content`.
///
/// # Example
///
/// ```
/// use tricc::lexer::Lexer;
///
/// let lexer = Lexer::new("let example: int = 4");
/// ```
pub fn new(content: &'a str) -> Self {
Lexer {
file,
text: contents,
chars: contents.chars().peekable(),
text: content,
chars: content.chars().peekable(),
tokens: VecDeque::new(),
line: 1,
start: 0,
end: 0,
@@ -120,49 +177,63 @@ impl<'a> Lexer<'a> {
}
#[inline]
fn error(&self) {
eprintln!("error lexing \"{}:{}:{}\"", self.file, self.line, self.end);
fn new_token(&self, kind: TokenKind) -> Token {
Token {
kind,
val: Rc::from(&self.text[self.start..self.end]),
}
}
#[inline]
fn error(&self, msg: &str) {
eprintln!("Lexer: {}, at \"{}:{}\"", msg, self.line, self.end);
}
#[inline]
fn peek(&mut self) -> Option<&char> {
self.chars.peek()
}
#[inline]
fn next(&mut self) -> Option<char> {
self.end += 1;
self.chars.next()
}
fn skip(&mut self, c: char) {
if self.next() != Some(c) {
self.error();
panic!("expected {}", c);
}
}
fn skip_whitespace(&mut self) {
let mut ignore_nl: bool = false;
fn escape_newline(&mut self) {
while let Some(c) = self.chars.peek() {
while let Some(c) = self.peek() {
match c {
'\r' | '\t' | ' ' => {
self.next();
}
'\n' => {
self.next();
break;
if ignore_nl {
ignore_nl = false;
self.next();
} else {
break;
}
}
_ => {
self.error();
panic!("expected newline");
},
'\\' => {
self.next();
ignore_nl = true;
}
_ => break,
}
}
}
fn get_numeric(&mut self) -> Token<'a> {
fn get_numeric(&mut self) -> Token {
let mut is_float: bool = false;
while let Some(c) = self.chars.peek() {
while let Some(c) = self.peek() {
match c {
'0'..='9' => {}
'.' => {
if is_float {
self.error();
panic!("multiple decimals encountered")
self.error("Multiple decimals encountered");
return self.new_token(TokenKind::Invalid);
}
is_float = true;
}
@@ -171,62 +242,78 @@ impl<'a> Lexer<'a> {
self.next();
}
Token::Literal(
if is_float {
TokenLiteral::Float
} else {
TokenLiteral::Int
},
&self.text[self.start..self.end],
)
self.new_token(TokenKind::Literal(if is_float {
TokenLiteral::Float
} else {
TokenLiteral::Int
}))
}
fn get_char(&mut self) -> Token<'a> {
self.skip('\'');
fn get_char(&mut self) -> Token {
// skip '
self.next();
if matches!(self.next(), Some('\'') | None) {
self.error();
panic!("A character literal cannot be empty");
self.error("Expected character literal");
return self.new_token(TokenKind::Invalid);
}
self.skip('\'');
// skip '
self.next();
Token::Literal(TokenLiteral::Char, &self.text[self.start + 1..self.end - 1])
self.new_token(TokenKind::Literal(TokenLiteral::Char))
}
fn get_delimiter(&mut self) -> Token<'a> {
use Token::Delimiter;
use TokenDelimiter::*;
match self.next() {
Some(c) => match c {
'{' => Delimiter(BraceOpen),
'}' => Delimiter(BraceClose),
'(' => Delimiter(ParenOpen),
')' => Delimiter(ParenClose),
_ => {
self.error();
panic!("expected delimiter");
}
},
None => {
self.error();
panic!("expected delimiter");
fn get_alphanumeric(&mut self) -> Token {
while let Some(c) = self.peek() {
match c {
'a'..='z' | 'A'..='Z' | '0'..='9' => {}
_ => break,
}
self.next();
}
use TokenKeyword::*;
use TokenKind::Keyword;
self.new_token(match &self.text[self.start..self.end] {
"fn" => Keyword(Fn),
"class" => Keyword(Class),
"module" => Keyword(Module),
"let" => Keyword(Let),
"ret" => Keyword(Ret),
"if" => Keyword(If),
"else" => Keyword(Else),
"elif" => Keyword(Elif),
"loop" => Keyword(Loop),
"break" => Keyword(Break),
"continue" => Keyword(Continue),
"int" => Keyword(Int),
"float" => Keyword(Float),
"char" => Keyword(Char),
_ => TokenKind::Identifier,
})
}
fn get_symbol(&mut self) -> Token<'a> {
use Token::Symbol;
fn get_symbol(&mut self) -> Token {
let c = self.next().unwrap();
use TokenDelimiter::*;
use TokenKind::{
Delimiter,
Symbol,
};
use TokenSymbol::*;
// handle +, +=, -, -=, *, *=, /, /=, %, %=, ^, ^=, !, !=
macro_rules! token_symbol_eq {
($a:expr, $b:expr) => {
if self.chars.peek() == Some(&'=') {
self.next();
Symbol($b)
} else {
Symbol($a)
match self.peek() {
Some('=') => {
self.next();
Symbol($b)
}
_ => Symbol($a),
}
};
}
@@ -234,7 +321,7 @@ impl<'a> Lexer<'a> {
// handle &, |, ||, &&, &=, |=
macro_rules! token_symbol_logical {
($a:expr, $b:expr, $c:expr, $d:expr) => {
match self.chars.peek() {
match self.peek() {
Some('=') => {
self.next();
Symbol($c)
@@ -251,7 +338,7 @@ impl<'a> Lexer<'a> {
// handle <, <=, >, >=, <<, >>, <<=, >>=
macro_rules! token_symbol_compare {
($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => {
match self.chars.peek() {
match self.peek() {
Some('=') => {
self.next();
Symbol($d)
@@ -265,99 +352,155 @@ impl<'a> Lexer<'a> {
};
}
use TokenSymbol::*;
match self.next() {
Some(c) => match c {
'+' => token_symbol_eq!(Plus, PlusEq),
'-' => token_symbol_eq!(Minus, MinusEq),
'*' => token_symbol_eq!(Star, StarEq),
'/' => token_symbol_eq!(Slash, SlashEq),
'%' => token_symbol_eq!(Percent, PercentEq),
'^' => token_symbol_eq!(Caret, CaretEq),
'!' => token_symbol_eq!(Not, Ne),
'=' => token_symbol_eq!(Eq, EqEq),
'&' => token_symbol_logical!(And, AndAnd, AndEq, '&'),
'|' => token_symbol_logical!(Or, OrOr, OrEq, '|'),
'<' => token_symbol_compare!(Lt, Shl, ShlEq, LtEq, '<'),
'>' => token_symbol_compare!(Gt, Shr, ShrEq, GtEq, '>'),
'~' => Symbol(Tilde),
':' => Symbol(Colon),
'.' => Symbol(Dot),
'#' => Symbol(Hash),
_ => {
self.error();
panic!("expected symbol");
}
},
None => {
self.error();
panic!("expected symbol");
let typ = match c {
'{' => Delimiter(BraceOpen),
'}' => Delimiter(BraceClose),
'(' => Delimiter(ParenOpen),
')' => Delimiter(ParenClose),
'+' => token_symbol_eq!(Plus, PlusEq),
'-' => token_symbol_eq!(Minus, MinusEq),
'*' => token_symbol_eq!(Star, StarEq),
'/' => token_symbol_eq!(Slash, SlashEq),
'%' => token_symbol_eq!(Percent, PercentEq),
'^' => token_symbol_eq!(Caret, CaretEq),
'!' => token_symbol_eq!(Not, Ne),
'=' => token_symbol_eq!(Eq, EqEq),
'&' => token_symbol_logical!(And, AndAnd, AndEq, '&'),
'|' => token_symbol_logical!(Or, OrOr, OrEq, '|'),
'<' => token_symbol_compare!(Lt, Shl, ShlEq, LtEq, '<'),
'>' => token_symbol_compare!(Gt, Shr, ShrEq, GtEq, '>'),
'~' => Symbol(Tilde),
':' => Symbol(Colon),
'.' => Symbol(Dot),
'#' => Symbol(Hash),
_ => {
self.error("Unknown character encountered");
TokenKind::Invalid
}
}
};
self.new_token(typ)
}
fn get_alphanumeric(&mut self) -> Token<'a> {
while let Some(c) = self.chars.peek() {
fn lex(&mut self) {
self.skip_whitespace();
self.start = self.end;
let token = if let Some(c) = self.peek() {
match c {
'a'..='z' | 'A'..='Z' | '0'..='9' => {}
_ => break,
}
self.next();
}
use Token::Keyword;
use TokenKeyword::*;
match &self.text[self.start..self.end] {
"let" => Keyword(Let),
"fn" => Keyword(Fn),
"ret" => Keyword(Ret),
"if" => Keyword(If),
"else" => Keyword(Else),
"elif" => Keyword(Elif),
"while" => Keyword(While),
"do" => Keyword(Do),
"for" => Keyword(For),
"int" => Keyword(Int),
"float" => Keyword(Float),
"char" => Keyword(Char),
_ => Token::Identifier(&self.text[self.start..self.end]),
}
}
pub fn lex(&mut self) -> Vec<Token<'a>> {
let mut tokens: Vec<Token> = Vec::new();
while let Some(c) = self.chars.peek() {
match c {
' ' | '\r' | '\t' => {
self.next();
}
'\\' => {
self.next();
self.escape_newline();
}
'\n' => {
tokens.push(Token::Newline);
self.next();
self.line += 1;
self.new_token(TokenKind::Newline)
}
'0'..='9' => tokens.push(self.get_numeric()),
'\'' => tokens.push(self.get_char()),
'{' | '}' | '(' | ')' => tokens.push(self.get_delimiter()),
'+' | '-' | '*' | '/' | '%' | '^' | '~' | '&' | '|' | '!' | '<' | '>' | '='
| ':' | '.' | '#' => tokens.push(self.get_symbol()),
'a'..='z' | 'A'..='Z' => tokens.push(self.get_alphanumeric()),
_ => {
self.error();
panic!("unknown character encountered");
}
'0'..='9' => self.get_numeric(),
'a'..='z' | 'A'..='Z' => self.get_alphanumeric(),
'\'' => self.get_char(),
_ => self.get_symbol(),
}
} else {
self.new_token(TokenKind::Eof)
};
self.tokens.push_back(token);
}
self.start = self.end;
/// Peeks at the next token and returns a reference to it
pub fn peek_token(&mut self) -> &Token {
if self.tokens.is_empty() {
self.lex();
}
&self.tokens[0]
}
tokens
/// Returns the next token, moving the lexer forward
pub fn next_token(&mut self) -> Token {
if self.tokens.is_empty() {
self.lex();
}
self.tokens.pop_front().unwrap()
}
}
#[test]
fn test_peek_next() {
let mut lexer = Lexer::new("test01");
assert_eq!(lexer.peek(), Some(&'t'));
assert_eq!(lexer.next(), Some('t'));
assert_eq!(lexer.peek(), Some(&'e'));
assert_eq!(lexer.peek(), Some(&'e'));
assert_eq!(lexer.next(), Some('e'));
assert_eq!(lexer.next(), Some('s'));
assert_eq!(lexer.next(), Some('t'));
assert_eq!(lexer.next(), Some('0'));
assert_eq!(lexer.peek(), Some(&'1'));
assert_eq!(lexer.next(), Some('1'));
assert_eq!(lexer.peek(), None);
assert_eq!(lexer.next(), None);
assert_eq!(lexer.peek(), None);
}
#[test]
fn test_tokens_1() {
let mut lexer = Lexer::new("let test02 = 4 << 1");
use TokenKind::*;
assert_eq!(lexer.peek_token().kind, Keyword(TokenKeyword::Let));
assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Let));
let mut token = lexer.next_token();
assert_eq!(token.kind, Identifier);
assert_eq!(*token.val, *"test02");
assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Eq));
token = lexer.next_token();
assert_eq!(token.kind, Literal(TokenLiteral::Int));
assert_eq!(*token.val, *"4");
assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Shl));
assert_eq!(lexer.peek_token().kind, Literal(TokenLiteral::Int));
assert_eq!(*lexer.peek_token().val, *"1");
token = lexer.next_token();
assert_eq!(token.kind, Literal(TokenLiteral::Int));
assert_eq!(*token.val, *"1");
assert_eq!(lexer.peek_token().kind, Eof);
assert_eq!(lexer.next_token().kind, Eof);
assert_eq!(lexer.peek_token().kind, Eof);
assert_eq!(lexer.next_token().kind, Eof);
}
#[test]
fn test_tokens_2() {
let mut lexer = Lexer::new("let test03: char = 'h'");
use TokenKind::*;
assert_eq!(lexer.peek_token().kind, Keyword(TokenKeyword::Let));
assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Let));
let mut token = lexer.next_token();
assert_eq!(token.kind, Identifier);
assert_eq!(*token.val, *"test03");
assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Colon));
assert_eq!(lexer.next_token().kind, Keyword(TokenKeyword::Char));
assert_eq!(lexer.next_token().kind, Symbol(TokenSymbol::Eq));
assert_eq!(lexer.peek_token().kind, Literal(TokenLiteral::Char));
assert_eq!(*lexer.peek_token().val, *"'h'");
token = lexer.next_token();
assert_eq!(token.kind, Literal(TokenLiteral::Char));
assert_eq!(*token.val, *"'h'");
assert_eq!(lexer.peek_token().kind, Eof);
assert_eq!(lexer.next_token().kind, Eof);
}
#[test]
fn test_tokens_3() {
let mut lexer = Lexer::new("");
assert_eq!(lexer.peek_token().kind, TokenKind::Eof);
assert_eq!(lexer.next_token().kind, TokenKind::Eof);
}

22
src/main.rs
View File

@@ -1,8 +1,13 @@
use std::fs;
use std::panic;
use std::{
fs,
panic,
};
use tricc::args::Args;
use tricc::lexer::Lexer;
use tricc::lexer::{
Lexer,
TokenKind,
};
fn main() {
panic::set_hook(Box::new(|panic_info| {
@@ -21,14 +26,15 @@ fn main() {
}
}));
let mut args = Args::new();
let mut args = Args::default();
args.handle();
let file = args.get_file();
let contents = fs::read_to_string(&file).expect("Couldn't read the file");
let content = fs::read_to_string(&file).expect("Couldn't read the file");
let mut lexer = Lexer::new(&file, contents.as_str());
let tokens = lexer.lex();
let mut lexer = Lexer::new(content.as_str());
println!("{:?}", tokens);
while lexer.peek_token().kind != TokenKind::Eof {
println!("{:?}", lexer.next_token());
}
}