Parser rewriting

core/parser/src/parser/mod.rs

@@ -6,6 +6,8 @@ mod statement;
 
 pub(crate) mod function;
 
+mod parse_loop;
+
 #[cfg(test)]
 mod tests;
 
@@ -15,6 +17,7 @@ use crate::{
     parser::{
         cursor::Cursor,
         function::{FormalParameters, FunctionStatementList},
+        parse_loop::{ControlFlow, ParseLoop, ParsedNode, SavedState, TokenLoopParser},
     },
     source::ReadChar,
     Error, Source,
@@ -387,15 +390,17 @@ where
     type Output = StatementList;
 
     fn parse(self, cursor: &mut Cursor<R>, interner: &mut Interner) -> ParseResult<Self::Output> {
-        let (body, _end) = statement::StatementList::new(
+        let entry = statement::StatementList::new(
             false,
             false,
             false,
             &[],
             self.directive_prologues,
             self.strict,
-        )
-        .parse(cursor, interner)?;
+        );
+        let stmt_list_node = ParseLoop::parse_loop(cursor, interner, entry)?;
+        let (body, _end) = (stmt_list_node.list, stmt_list_node.pos);
+        // let (body, _end) = entry.parse(cursor, interner)?;
 
         if !self.direct_eval {
             // It is a Syntax Error if StatementList Contains super unless the source text containing super is eval

core/parser/src/parser/parse_loop.rs

@@ -0,0 +1,217 @@
+use boa_ast::declaration::LexicalDeclaration;
+use boa_ast::Position;
+use boa_interner::Interner;
+
+use boa_ast as ast;
+use crate::error::{ParseResult, Error};
+use crate::lexer::Token;
+use crate::source::ReadChar;
+use crate::parser::Cursor;
+use crate::parser::statement::{StatementList, StatementListLocal, StatementListNode};
+
+#[macro_export]
+macro_rules! parse_cmd {
+    // or move into `pop_local_state!`
+    [[POP LOCAL]: $state:ident => Empty] => {{
+        let Ok($crate::parser::SavedState::Empty) = $state.pop_local_state() else {
+            return Err($state.general_error(concat!("expect `Empty` saved state")))
+        };
+    }};
+    // or move into `pop_local_state!`
+    [[POP LOCAL]: $state:ident => $variant:ident] => {{
+        let Ok($crate::parser::SavedState::$variant(ret)) = $state.pop_local_state() else {
+            return Err($state.general_error(concat!("expect `", stringify!($variant) ,"Local` saved state")))
+        };
+        ret
+    }};
+
+    // or move into `pop_node!`
+    [[POP NODE]: $state:ident => $variant:ident] => {{
+        let Ok($crate::parser::ParsedNode::$variant(ret)) = $state.pop_node() else {
+            return Err($state.general_error(concat!("expect `", stringify!($variant) ,"` node")))
+        };
+        ret
+    }};
+
+    // or move into `peek_node!`
+    [[PEEK NODE]: $state:ident match $($variants:ident)+] => {{
+        if !matches!($state.peek_node(), Some($(| $crate::parser::ParsedNode::$variants(_))+)) {
+            return Err($state.general_error(concat!("expect `", $(stringify!($variants), "` | `",)+ "` node")))
+        }
+    }};
+
+    // or move into `sub_parse!`
+    [[SUB PARSE]: $item:expr; $state:ident <= $local:ident as $variant:ident ($point:literal)] => {{
+        $state.push_local($crate::parser::SavedState::$variant($local));
+        return ParseResult::Ok($crate::parser::ControlFlow::SubParse { node: Box::new($item), point: $point });
+    }};
+    // or move into `sub_parse!`
+    [[SUB PARSE]: $item:expr; $state:ident <= Empty ($point:literal)] => {{
+        $state.push_local($crate::parser::SavedState::Empty);
+        return ParseResult::Ok($crate::parser::ControlFlow::SubParse { node: Box::new($item), point: $point });
+    }};
+
+    // or move into `parse_done!`
+    [[DONE]: $state:ident <= $variant:ident($node:expr)] => {{
+        $state.push_node($crate::parser::ParsedNode::$variant($node));
+        return Ok($crate::parser::ControlFlow::Done)
+    }};
+}
+
+pub(super) struct ParseLoop;
+
+impl ParseLoop {
+    pub(super) fn parse_loop<R: ReadChar>(
+        cursor: &mut Cursor<R>,
+        interner: &mut Interner,
+        entry: StatementList
+    ) -> ParseResult<StatementListNode> {
+        let mut state: ParseState<'_, R> = ParseState::new(cursor, interner);
+
+        let mut parse_stack: Vec<Box<dyn TokenLoopParser<R>>> = vec![Box::new(entry)];
+        let mut continue_points = vec![0];
+
+        loop {
+            debug_assert!(!parse_stack.is_empty());
+            debug_assert_eq!(continue_points.len(), parse_stack.len());
+
+            // SAFETY:
+            // we push (entry, 0) on first iteration & it will pop only on
+            // last `ControlFlow::Done` after which this fuction returns
+            let continue_point = continue_points.pop().unwrap();
+            let parser = parse_stack.last_mut().unwrap();
+
+            match parser.parse_loop(&mut state, continue_point)? {
+                ControlFlow::SubParse { node, point } => {
+                    continue_points.push(point); // reinsert current updated `continue_point`
+                    continue_points.push(0); // insert continue point for new sub parsing node
+
+                    parse_stack.push(node);
+                }
+                ControlFlow::Done => {
+                    // remove parsing node from stack (`continue_point` already removed)
+                    parse_stack.pop();
+
+                    if parse_stack.is_empty() {
+                        let stmt_list_node = parse_cmd![[POP NODE]: state => StatementList];
+                        assert!(state.nodes.is_empty());
+                        return Ok(stmt_list_node)
+                    }
+                }
+            }
+
+        }
+    }
+}
+
+/// Trait implemented by parsers.
+///
+/// This makes it possible to abstract over the underlying implementation of a parser.
+pub(super) trait TokenLoopParser<R>
+where
+    R: ReadChar,
+{
+    /// Parses the token stream using the current parser.
+    ///
+    /// This method needs to be provided by the implementor type.
+    ///
+    /// # Errors
+    ///
+    /// It will fail if the cursor is not placed at the beginning of the expected non-terminal.
+    fn parse_loop(&mut self, state: &mut ParseState<'_, R>, continue_point: usize) -> ParseResult<ControlFlow<R>>;
+}
+
+pub(super) enum ControlFlow<R>
+where R: ReadChar,
+{
+    SubParse{node: Box<dyn TokenLoopParser<R>>, point: usize},
+    Done,
+}
+
+pub(super) struct ParseState<'a, R> {
+    nodes: Vec<ParsedNode>,
+    saved_state: Vec<SavedState>,
+    pub cursor: &'a mut Cursor<R>,
+    pub interner: &'a mut Interner,
+}
+impl<'a, R: ReadChar> ParseState<'a, R> {
+    pub(super) fn new(cursor: &'a mut Cursor<R>, interner: &'a mut Interner) -> Self {
+        Self {
+            nodes: Vec::new(),
+            saved_state: Vec::new(),
+            cursor,
+            interner,
+        }
+    }
+    pub(super) fn mut_inner(&mut self) -> (&mut Cursor<R>, &mut Interner) {
+        (&mut self.cursor, &mut self.interner)
+    }
+
+    pub(super) fn cursor(&mut self) -> &Cursor<R> {
+        &self.cursor
+    }
+    pub(super) fn cursor_mut(&mut self) -> &mut Cursor<R> {
+        &mut self.cursor
+    }
+    pub(super) fn interner(&self) -> &Interner {
+        &self.interner
+    }
+    pub(super) fn interner_mut(&mut self) -> &mut Interner {
+        &mut self.interner
+    }
+
+    pub(super) fn push_node(&mut self, node: ParsedNode) {
+        self.nodes.push(node);
+    }
+    pub(super) fn push_local(&mut self, local: SavedState) {
+        self.saved_state.push(local);
+    }
+
+    pub(super) fn pop_node(&mut self) -> ParseResult<ParsedNode> {
+        self.nodes.pop().ok_or_else(||self.general_error("expect parsed node"))
+    }
+    pub(super) fn peek_node(&mut self) -> Option<&ParsedNode> {
+        self.nodes.last()
+    }
+
+    pub(super) fn pop_local_state(&mut self) -> ParseResult<SavedState> {
+        self.saved_state.pop().ok_or_else(||self.general_error("expect saved state"))
+    }
+
+    pub(super) fn continue_point_error<T>(&self, continue_point: usize) -> ParseResult<T> {
+        Err(self.general_error(format!("unexpected continue point ({continue_point})")))
+    }
+
+    pub(super) fn general_error<S: AsRef<str>>(&self, msg: S) -> Error {
+        Error::general(
+            format!("{}; linear position: {}", msg.as_ref(), self.cursor.linear_pos()),
+            Position::new(1, 1) // TODO: something to take last position see `self.cursor.linear_pos()`
+        )
+    }
+
+    ///Peeks a future token, without consuming it or advancing the cursor. This peeking skips line terminators.
+    ///
+    /// You can skip some tokens with the `skip_n` option.
+    pub(super) fn peek(&mut self, skip_n: usize) -> ParseResult<Option<&Token>> {
+        self.cursor.peek(skip_n, &mut self.interner)
+    }
+
+    /// Check if the peeked token is a line terminator.
+    pub(super) fn peek_is_line_terminator(&mut self, skip_n: usize) -> ParseResult<Option<bool>> {
+        self.cursor.peek_is_line_terminator(skip_n, &mut self.interner)
+    }
+
+}
+
+pub(super) enum ParsedNode {
+    Empty,
+    StatementListItem(ast::StatementListItem),
+    StatementList(StatementListNode),
+    Declaration(ast::Declaration),
+    Statement(ast::Statement),
+}
+
+pub(super) enum SavedState {
+    Empty,
+    StatementList(StatementListLocal),
+}

core/parser/src/parser/statement/declaration/hoistable/mod.rs

@@ -27,6 +27,7 @@ use crate::{
     source::ReadChar,
     Error,
 };
+use crate::{parse_cmd, parser::{ControlFlow, TokenLoopParser, parse_loop::ParseState}};
 use boa_ast::{
     self as ast,
     expression::Identifier,
@@ -125,6 +126,14 @@ where
     }
 }
 
+impl<R: ReadChar> TokenLoopParser<R> for HoistableDeclaration {
+    fn parse_loop(&mut self, state: &mut ParseState<'_, R>, _continue_point: usize) -> ParseResult<ControlFlow<R>> {
+        let (cursor, interner) = state.mut_inner();
+        let ok = self.parse(cursor, interner)?;
+        parse_cmd!([DONE]: state <= Declaration(ok))
+    }
+}
+
 trait CallableDeclaration {
     fn error_context(&self) -> &'static str;
     fn is_default(&self) -> bool;

core/parser/src/parser/statement/declaration/lexical.rs

@@ -18,6 +18,7 @@ use crate::{
     source::ReadChar,
     Error,
 };
+use crate::{parse_cmd, parser::{ControlFlow, TokenLoopParser, parse_loop::ParseState}};
 use ast::operations::bound_names;
 use boa_ast::{self as ast, declaration::Variable, Keyword, Punctuator};
 use boa_interner::{Interner, Sym};
@@ -121,6 +122,14 @@ where
     }
 }
 
+impl<R: ReadChar> TokenLoopParser<R> for LexicalDeclaration {
+    fn parse_loop(&mut self, state: &mut ParseState<'_, R>, _continue_point: usize) -> ParseResult<ControlFlow<R>> {
+        let (cursor, interner) = state.mut_inner();
+        let ok = self.parse(cursor, interner)?;
+        parse_cmd!([DONE]: state <= Declaration(ok.into()))
+    }
+}
+
 /// Check if the given token is valid after the `let` keyword of a lexical declaration.
 pub(crate) fn allowed_token_after_let(token: Option<&Token>) -> bool {
     matches!(

core/parser/src/parser/statement/declaration/mod.rs

@@ -23,10 +23,7 @@ pub(in crate::parser) use self::{
     lexical::{allowed_token_after_let, LexicalDeclaration},
 };
 use crate::{
-    lexer::TokenKind,
-    parser::{AllowAwait, AllowYield, Cursor, OrAbrupt, ParseResult, TokenParser},
-    source::ReadChar,
-    Error,
+    lexer::TokenKind, parse_cmd, parser::{parse_loop::ParseState, AllowAwait, AllowYield, ControlFlow, Cursor, OrAbrupt, ParseResult, TokenLoopParser, TokenParser}, source::ReadChar, Error
 };
 use boa_ast::{self as ast, Keyword};
 use boa_interner::{Interner, Sym};
@@ -93,6 +90,43 @@ where
     }
 }
 
+impl<R: ReadChar> TokenLoopParser<R> for Declaration {
+    fn parse_loop(&mut self, state: &mut ParseState<'_, R>, continue_point: usize) -> ParseResult<ControlFlow<R>> {
+        if continue_point == 1 {
+            parse_cmd![[POP LOCAL]: state => Empty];
+            parse_cmd![[PEEK NODE]: state match Declaration]; // pass value up
+            return Ok(ControlFlow::Done)
+        } else if continue_point > 1 {
+            return state.continue_point_error(continue_point)
+        }
+
+        let tok = state.cursor.peek(0, state.interner).or_abrupt()?;
+
+        match tok.kind() {
+            TokenKind::Keyword((Keyword::Function | Keyword::Async | Keyword::Class, _)) => {
+                let node = HoistableDeclaration::new(self.allow_yield, self.allow_await, false);
+                parse_cmd![[SUB PARSE]: node; state <= Empty (1)];
+            }
+            TokenKind::Keyword((Keyword::Const | Keyword::Let, _)) => {
+                let node = LexicalDeclaration::new(true, self.allow_yield, self.allow_await, false);
+                parse_cmd![[SUB PARSE]: node; state <= Empty (1)];
+            }
+            _ => return Err(Error::expected(
+                [
+                    Keyword::Function.to_string(),
+                    Keyword::Async.to_string(),
+                    Keyword::Class.to_string(),
+                    Keyword::Const.to_string(),
+                    Keyword::Let.to_string(),
+                ],
+                tok.to_string(state.interner),
+                tok.span(),
+                "export declaration",
+            )),
+        }
+    }
+}
+
 /// Parses a `from` clause.
 ///
 /// More information: