parser: fix newline checks after statements

since there is no EOL delimiter like ';', we have to check for newline to avoid something like `a = 4 + 4 b = a`
also added tests for expr parsing

Signed-off-by: Amneesh Singh <natto@weirdnatto.in>

src/ast.rs

@@ -12,7 +12,7 @@ pub enum Entity {
     Fn(Fn),
     Class(Class),
     Module(Module),
-    Static(Let)
+    Static(Let),
 }
 
 /// A module just provides an additional scope
@@ -106,7 +106,7 @@ pub enum ElseType {
     Else(Vec<Statement>),
 }
 
-type Op = crate::lexer::TokenSymbol;
+pub(crate) type Op = crate::lexer::TokenSymbol;
 
 #[derive(Debug, PartialEq)]
 pub enum Literal {

src/lexer.rs

@@ -6,7 +6,7 @@ use std::str;
 /// All token literals
 ///
 /// TODO: Add string
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum TokenLiteral {
     Int,
     Float,
@@ -65,7 +65,7 @@ pub enum TokenSymbol {
 }
 
 /// All token keywod
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum TokenKeyword {
     // parents
     Fn,
@@ -97,7 +97,7 @@ pub enum TokenKeyword {
 /// All token delimiters
 ///
 /// TODO: Maybe add \[ and \]
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum TokenDelimiter {
     BraceOpen,
     BraceClose,
@@ -106,7 +106,7 @@ pub enum TokenDelimiter {
 }
 
 /// All tokens
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum TokenKind {
     Newline,
     Eof,

src/parser/entity.rs

@@ -57,6 +57,10 @@ impl<'a> Parser<'a> {
                         return None;
                     }
                 });
+                if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
+                    self.error_expected_peek("newline or }");
+                    return None;
+                }
             } else if !self.skip_token(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
                 self.error_expected_peek("}");
                 return None;
@@ -93,6 +97,10 @@ impl<'a> Parser<'a> {
                         return None;
                     }
                 });
+                if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
+                    self.error_expected_peek("newline or }");
+                    return None;
+                }
             } else if !self.skip_token(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
                 self.error_expected_peek("}");
                 return None;
@@ -148,7 +156,11 @@ impl<'a> Parser<'a> {
             if self.skip_token(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
                 break;
             }
-            children.push(self.parse_statement()?)
+            children.push(self.parse_statement()?);
+            if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
+                self.error_expected_peek("newline or }");
+                return None;
+            }
         }
 
         Some(Fn {

src/parser/expr.rs

@@ -57,6 +57,10 @@ impl<'a> Parser<'a> {
                 break;
             }
             statements.push(self.parse_statement()?);
+            if !self.check_newline_or_tok(TokenKind::Delimiter(TokenDelimiter::BraceClose)) {
+                self.error_expected_peek("newline or }");
+                return None;
+            }
         }
 
         Some(statements)
@@ -325,32 +329,107 @@ impl<'a> Parser<'a> {
     ///
     /// <https://en.wikipedia.org/wiki/Recursive_descent_parser>
     /// expr ::= exprControl
-    fn parse_expr(&mut self) -> Option<Expr> {
+    pub(super) fn parse_expr(&mut self) -> Option<Expr> {
         self.parse_expr_control()
     }
-
-    pub(super) fn parse_expr_ln(&mut self) -> Option<Expr> {
-        let expr = self.parse_expr();
-        if !self.skip_token(TokenKind::Newline) {
-            self.error_expected_peek("newline");
-            return None;
-        }
-        expr
-    }
 }
 
 #[test]
-fn test_parse_expr_literals() {
+fn test_parse_expr() {
     use Literal::*;
+    use TokenSymbol::*;
+
+    macro_rules! b {
+        ($expr:expr) => {
+            Box::new($expr)
+        };
+    }
+
+    let mut parser = Parser::new(
+        r#"if if 1 { 1 } else { 0 } + 9 {
+             a = 4
+           } else if 1 {
+             a = 5
+           } else {
+           }
+          amul ^= (4 + 93 * (1 << 3) / 1.44) ^ bhatura
+          stove = { 44 } + amul"#,
+    );
 
-    let mut parser = Parser::new("4524 3123.15e4 9e2 9083482.429455 'c' 3331.13.3");
-    assert_eq!(parser.parse_expr(), Some(Expr::Literal(Int(4524))));
-    assert_eq!(parser.parse_expr(), Some(Expr::Literal(Float(3123.15e4))));
-    assert_eq!(parser.parse_expr(), Some(Expr::Literal(Float(9e2))));
     assert_eq!(
         parser.parse_expr(),
-        Some(Expr::Literal(Float(9083482.429455)))
+        Some(Expr::If(If {
+            cond: b!(Expr::Op(
+                Plus,
+                b!(Expr::If(If {
+                    cond: b!(Expr::Literal(Int(1))),
+                    then: vec![Statement::Expr(Expr::Literal(Int(1)))],
+                    or: Some(b!(ElseType::Else(vec![Statement::Expr(Expr::Literal(
+                        Int(0)
+                    ))])))
+                })),
+                Some(b!(Expr::Literal(Int(9))))
+            )),
+            then: vec![Statement::Expr(Expr::Op(
+                Eq,
+                b!(Expr::Identifier("a".into())),
+                Some(b!(Expr::Literal(Int(4))))
+            ))],
+            or: Some(b!(ElseType::If(If {
+                cond: b!(Expr::Literal(Int(1))),
+                then: vec![Statement::Expr(Expr::Op(
+                    Eq,
+                    b!(Expr::Identifier("a".into())),
+                    Some(b!(Expr::Literal(Int(5))))
+                ))],
+                or: Some(b!(ElseType::Else(vec![])))
+            })))
+        }))
+    );
+
+    assert_eq!(parser.skip_token(TokenKind::Newline), true);
+
+    assert_eq!(
+        parser.parse_expr(),
+        Some(Expr::Op(
+            CaretEq,
+            b!(Expr::Identifier("amul".into())),
+            Some(b!(Expr::Op(
+                Caret,
+                b!(Expr::Op(
+                    Plus,
+                    b!(Expr::Literal(Int(4))),
+                    Some(b!(Expr::Op(
+                        Star,
+                        b!(Expr::Literal(Int(93))),
+                        Some(b!(Expr::Op(
+                            Slash,
+                            b!(Expr::Op(
+                                Shl,
+                                b!(Expr::Literal(Int(1))),
+                                Some(b!(Expr::Literal(Int(3))))
+                            )),
+                            Some(b!(Expr::Literal(Float(1.44))))
+                        )))
+                    )))
+                )),
+                Some(b!(Expr::Identifier("bhatura".into())))
+            )))
+        ))
+    );
+
+    assert_eq!(parser.skip_token(TokenKind::Newline), true);
+
+    assert_eq!(
+        parser.parse_expr(),
+        Some(Expr::Op(
+            Eq,
+            b!(Expr::Identifier("stove".into())),
+            Some(b!(Expr::Op(
+                Plus,
+                b!(Expr::Block(vec![Statement::Expr(Expr::Literal(Int(44)))])),
+                Some(b!(Expr::Identifier("amul".into())))
+            )))
+        ))
     );
-    assert_eq!(parser.parse_expr(), Some(Expr::Literal(Char('c'))));
-    assert_eq!(parser.parse_expr(), None);
 }

src/parser/literal.rs

@@ -108,3 +108,14 @@ impl<'a> Parser<'a> {
         self.next_token().val.chars().nth(1)
     }
 }
+
+#[test]
+fn test_parse_literals() {
+    let mut parser = Parser::new("4524 3123.15e4 9e2 9083482.429455 'c' 3331.13.1");
+    assert_eq!(parser.parse_int(), Some(4524));
+    assert_eq!(parser.parse_float(), Some(3123.15e4));
+    assert_eq!(parser.parse_float(), Some(9e2));
+    assert_eq!(parser.parse_float(), Some(9083482.429455));
+    assert_eq!(parser.parse_char(), Some('c'));
+    assert_eq!(parser.next_token().kind, TokenKind::Invalid);
+}

src/parser/mod.rs

@@ -53,7 +53,9 @@ impl<'a> Parser<'a> {
 
     #[inline]
     fn next_token(&mut self) -> Token {
-        self.lexer.next_token()
+        let t = self.lexer.next_token();
+        println!("{:?}", t);
+        t
     }
 
     #[inline]
@@ -61,7 +63,7 @@ impl<'a> Parser<'a> {
         return self.lexer.peek_token();
     }
 
-    /// newline ::= "\n"
+    /// newline ::= "}\n"
     fn trim_newlines(&mut self) {
         while self.peek_token().kind == TokenKind::Newline {
             self.next_token();
@@ -77,6 +79,14 @@ impl<'a> Parser<'a> {
         false
     }
 
+    fn check_newline_or_tok(&mut self, token: TokenKind) -> bool {
+        match self.peek_token().kind {
+            TokenKind::Newline => true,
+            d if d == token => true,
+            _ => false,
+        }
+    }
+
     /// ty ::= "int" | "float" | "char"
     fn parse_ty(&mut self) -> Option<Ty> {
         let ty: Ty;
@@ -134,6 +144,10 @@ impl<'a> Parser<'a> {
                 TokenKind::Eof => break,
                 _ => {
                     parent.push(self.parse_entity()?);
+                    if !self.check_newline_or_tok(TokenKind::Eof) {
+                        self.error_expected_peek("newline or end of file");
+                        return None;
+                    }
                 }
             }
         }

src/parser/statement.rs

@@ -10,11 +10,12 @@ impl<'a> Parser<'a> {
     /// statement ::= static | let | expr
     pub(super) fn parse_statement(&mut self) -> Option<Statement> {
         use TokenKeyword::*;
+        println!("STMT");
 
         Some(match self.peek_token().kind {
             TokenKind::Keyword(Static) => Statement::Static(self.parse_static()?),
             TokenKind::Keyword(Let) => Statement::Let(self.parse_let()?),
-            _ => Statement::Expr(self.parse_expr_ln()?),
+            _ => Statement::Expr(self.parse_expr()?),
         })
     }
 
@@ -37,8 +38,8 @@ impl<'a> Parser<'a> {
         let (name, ty) = self.parse_ident_with_ty()?;
 
         let expr = if self.skip_token(TokenKind::Symbol(TokenSymbol::Eq)) {
-            self.parse_expr_ln()
+            self.parse_expr()
-        } else if self.skip_token(TokenKind::Newline) {
+        } else if self.peek_token().kind == TokenKind::Newline {
             None
         } else {
             self.error_expected_peek("= or newline");
@@ -67,6 +68,7 @@ fn test_parse_let() {
             expr: Some(Expr::Literal(Int(4)))
         })
     );
+    assert_eq!(parser.skip_token(TokenKind::Newline), true);
     assert_eq!(
         parser.parse_let(),
         Some(Let {
@@ -75,6 +77,7 @@ fn test_parse_let() {
             expr: Some(Expr::Literal(Char('6')))
         })
     );
+    assert_eq!(parser.skip_token(TokenKind::Newline), true);
     assert_eq!(
         parser.parse_static(),
         Some(Let {
@@ -83,5 +86,6 @@ fn test_parse_let() {
             expr: None
         })
     );
+    assert_eq!(parser.skip_token(TokenKind::Newline), true);
     assert_eq!(parser.parse_let(), None);
 }