natto1784/tricc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

parser: fix newline checks after statements

Browse Source 
since there is no EOL delimiter like ';', we have to check for newline to avoid something like `a = 4 + 4 b = a`
also added tests for expr parsing

Signed-off-by: Amneesh Singh <natto@weirdnatto.in>
This commit is contained in:
Amneesh Singh
2023-08-14 00:56:09 +05:30
parent 360687f3c0
commit af30410aab
7 changed files with 150 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/ast.rs
View File

@@ -12,7 +12,7 @@ pub enum Entity {
Fn(Fn),
Class(Class),
Module(Module),
Static(Let)
Static(Let),
}
/// A module just provides an additional scope
@@ -106,7 +106,7 @@ pub enum ElseType {
Else(Vec<Statement>),
}
type Op = crate::lexer::TokenSymbol;
pub(crate) type Op = crate::lexer::TokenSymbol;
#[derive(Debug, PartialEq)]
pub enum Literal {

8
src/lexer.rs
View File

@@ -6,7 +6,7 @@ use std::str;
/// All token literals
///
/// TODO: Add string
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum TokenLiteral {
Int,
Float,
@@ -65,7 +65,7 @@ pub enum TokenSymbol {
}
/// All token keywod
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum TokenKeyword {
// parents
Fn,
@@ -97,7 +97,7 @@ pub enum TokenKeyword {
/// All token delimiters
///
/// TODO: Maybe add \[ and \]
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum TokenDelimiter {
BraceOpen,
BraceClose,
@@ -106,7 +106,7 @@ pub enum TokenDelimiter {
}
/// All tokens
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum TokenKind {
Newline,
Eof,

14
src/parser/entity.rs
View File

@@ -57,6 +57,10 @@ impl<'a> Parser<'a> {
return None;
}
});
if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
self.error_expected_peek("newline or }");
return None;
}
} else if !self.skip_token(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
self.error_expected_peek("}");
return None;
@@ -93,6 +97,10 @@ impl<'a> Parser<'a> {
return None;
}
});
if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
self.error_expected_peek("newline or }");
return None;
}
} else if !self.skip_token(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
self.error_expected_peek("}");
return None;
@@ -148,7 +156,11 @@ impl<'a> Parser<'a> {
if self.skip_token(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
break;
}
children.push(self.parse_statement()?)
children.push(self.parse_statement()?);
if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
self.error_expected_peek("newline or }");
return None;
}
}
Some(Fn {

115
src/parser/expr.rs
View File

@@ -57,6 +57,10 @@ impl<'a> Parser<'a> {
break;
}
statements.push(self.parse_statement()?);
if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
self.error_expected_peek("newline or }");
return None;
}
}
Some(statements)
@@ -325,32 +329,107 @@ impl<'a> Parser<'a> {
///
/// <https://en.wikipedia.org/wiki/Recursive_descent_parser>
/// expr ::= exprControl
fn parse_expr(&mut self) -> Option<Expr> {
pub(super) fn parse_expr(&mut self) -> Option<Expr> {
self.parse_expr_control()
}
pub(super) fn parse_expr_ln(&mut self) -> Option<Expr> {
let expr = self.parse_expr();
if !self.skip_token(TokenKind::Newline) {
self.error_expected_peek("newline");
return None;
}
expr
}
}
#[test]
fn test_parse_expr_literals() {
fn test_parse_expr() {
use Literal::*;
use TokenSymbol::*;
macro_rules! b {
($expr:expr) => {
Box::new($expr)
};
}
let mut parser = Parser::new(
r#"if if 1 { 1 } else { 0 } + 9 {
a = 4
} else if 1 {
a = 5
} else {
}
amul ^= (4 + 93 * (1 << 3) / 1.44) ^ bhatura
stove = { 44 } + amul"#,
);
let mut parser = Parser::new("4524 3123.15e4 9e2 9083482.429455 'c' 3331.13.3");
assert_eq!(parser.parse_expr(), Some(Expr::Literal(Int(4524))));
assert_eq!(parser.parse_expr(), Some(Expr::Literal(Float(3123.15e4))));
assert_eq!(parser.parse_expr(), Some(Expr::Literal(Float(9e2))));
assert_eq!(
parser.parse_expr(),
Some(Expr::Literal(Float(9083482.429455)))
Some(Expr::If(If {
cond: b!(Expr::Op(
Plus,
b!(Expr::If(If {
cond: b!(Expr::Literal(Int(1))),
then: vec![Statement::Expr(Expr::Literal(Int(1)))],
or: Some(b!(ElseType::Else(vec![Statement::Expr(Expr::Literal(
Int(0)
))])))
})),
Some(b!(Expr::Literal(Int(9))))
)),
then: vec![Statement::Expr(Expr::Op(
Eq,
b!(Expr::Identifier("a".into())),
Some(b!(Expr::Literal(Int(4))))
))],
or: Some(b!(ElseType::If(If {
cond: b!(Expr::Literal(Int(1))),
then: vec![Statement::Expr(Expr::Op(
Eq,
b!(Expr::Identifier("a".into())),
Some(b!(Expr::Literal(Int(5))))
))],
or: Some(b!(ElseType::Else(vec![])))
})))
}))
);
assert_eq!(parser.skip_token(TokenKind::Newline), true);
assert_eq!(
parser.parse_expr(),
Some(Expr::Op(
CaretEq,
b!(Expr::Identifier("amul".into())),
Some(b!(Expr::Op(
Caret,
b!(Expr::Op(
Plus,
b!(Expr::Literal(Int(4))),
Some(b!(Expr::Op(
Star,
b!(Expr::Literal(Int(93))),
Some(b!(Expr::Op(
Slash,
b!(Expr::Op(
Shl,
b!(Expr::Literal(Int(1))),
Some(b!(Expr::Literal(Int(3))))
)),
Some(b!(Expr::Literal(Float(1.44))))
)))
)))
)),
Some(b!(Expr::Identifier("bhatura".into())))
)))
))
);
assert_eq!(parser.skip_token(TokenKind::Newline), true);
assert_eq!(
parser.parse_expr(),
Some(Expr::Op(
Eq,
b!(Expr::Identifier("stove".into())),
Some(b!(Expr::Op(
Plus,
b!(Expr::Block(vec![Statement::Expr(Expr::Literal(Int(44)))])),
Some(b!(Expr::Identifier("amul".into())))
)))
))
);
assert_eq!(parser.parse_expr(), Some(Expr::Literal(Char('c'))));
assert_eq!(parser.parse_expr(), None);
}

11
src/parser/literal.rs
View File

@@ -108,3 +108,14 @@ impl<'a> Parser<'a> {
self.next_token().val.chars().nth(1)
}
}
#[test]
fn test_parse_literals() {
let mut parser = Parser::new("4524 3123.15e4 9e2 9083482.429455 'c' 3331.13.1");
assert_eq!(parser.parse_int(), Some(4524));
assert_eq!(parser.parse_float(), Some(3123.15e4));
assert_eq!(parser.parse_float(), Some(9e2));
assert_eq!(parser.parse_float(), Some(9083482.429455));
assert_eq!(parser.parse_char(), Some('c'));
assert_eq!(parser.next_token().kind, TokenKind::Invalid);
}

18
src/parser/mod.rs
View File

@@ -53,7 +53,9 @@ impl<'a> Parser<'a> {
#[inline]
fn next_token(&mut self) -> Token {
self.lexer.next_token()
let t = self.lexer.next_token();
println!("{:?}", t);
t
}
#[inline]
@@ -61,7 +63,7 @@ impl<'a> Parser<'a> {
return self.lexer.peek_token();
}
/// newline ::= "\n"
/// newline ::= "}\n"
fn trim_newlines(&mut self) {
while self.peek_token().kind == TokenKind::Newline {
self.next_token();
@@ -77,6 +79,14 @@ impl<'a> Parser<'a> {
false
}
fn check_newline_or_tok(&mut self, token: TokenKind) -> bool {
match self.peek_token().kind {
TokenKind::Newline => true,
d if d == token => true,
_ => false,
}
}
/// ty ::= "int" | "float" | "char"
fn parse_ty(&mut self) -> Option<Ty> {
let ty: Ty;
@@ -134,6 +144,10 @@ impl<'a> Parser<'a> {
TokenKind::Eof => break,
_ => {
parent.push(self.parse_entity()?);
if !self.check_newline_or_tok(TokenKind::Eof) {
self.error_expected_peek("newline or end of file");
return None;
}
}
}
}

10
src/parser/statement.rs
View File

@@ -10,11 +10,12 @@ impl<'a> Parser<'a> {
/// statement ::= static | let | expr
pub(super) fn parse_statement(&mut self) -> Option<Statement> {
use TokenKeyword::*;
println!("STMT");
Some(match self.peek_token().kind {
TokenKind::Keyword(Static) => Statement::Static(self.parse_static()?),
TokenKind::Keyword(Let) => Statement::Let(self.parse_let()?),
_ => Statement::Expr(self.parse_expr_ln()?),
_ => Statement::Expr(self.parse_expr()?),
})
}
@@ -37,8 +38,8 @@ impl<'a> Parser<'a> {
let (name, ty) = self.parse_ident_with_ty()?;
let expr = if self.skip_token(TokenKind::Symbol(TokenSymbol::Eq)) {
self.parse_expr_ln()
} else if self.skip_token(TokenKind::Newline) {
self.parse_expr()
} else if self.peek_token().kind == TokenKind::Newline {
None
} else {
self.error_expected_peek("= or newline");
@@ -67,6 +68,7 @@ fn test_parse_let() {
expr: Some(Expr::Literal(Int(4)))
})
);
assert_eq!(parser.skip_token(TokenKind::Newline), true);
assert_eq!(
parser.parse_let(),
Some(Let {
@@ -75,6 +77,7 @@ fn test_parse_let() {
expr: Some(Expr::Literal(Char('6')))
})
);
assert_eq!(parser.skip_token(TokenKind::Newline), true);
assert_eq!(
parser.parse_static(),
Some(Let {
@@ -83,5 +86,6 @@ fn test_parse_let() {
expr: None
})
);
assert_eq!(parser.skip_token(TokenKind::Newline), true);
assert_eq!(parser.parse_let(), None);
}