Correctly parse consecutive semicolons (#2533)

This Pull Request changes the following:

- Do not skip consecutive semicolons while parsing a `StatementList`.
- Expect semicolon in `LexicalDeclaration` and add an special case for `for` loop parsing.
- Adjust `StatementList` compilation to skip empty statements.
- Adjust/add tests to make sure consecutive semicolons  are correctly parsed.

boa_engine/src/bytecompiler/mod.rs

@@ -660,12 +660,20 @@ impl<'b, 'icu> ByteCompiler<'b, 'icu> {
         use_expr: bool,
         configurable_globals: bool,
     ) -> JsResult<()> {
-        if let Some((last, items)) = list.statements().split_last() {
-            for node in items {
-                self.compile_stmt_list_item(node, false, configurable_globals)?;
+        let mut items = list
+            .statements()
+            .iter()
+            .filter(|item| item != &&StatementListItem::Statement(Statement::Empty))
+            .peekable();
+
+        while let Some(item) = items.next() {
+            if items.peek().is_some() {
+                self.compile_stmt_list_item(item, false, configurable_globals)?;
+            } else {
+                self.compile_stmt_list_item(item, use_expr, configurable_globals)?;
             }
-            self.compile_stmt_list_item(last, use_expr, configurable_globals)?;
         }
+
         Ok(())
     }
 
@@ -681,11 +689,18 @@ impl<'b, 'icu> ByteCompiler<'b, 'icu> {
 
         self.create_decls(list, true);
 
-        if let Some((last, items)) = list.statements().split_last() {
-            for node in items {
-                self.compile_stmt_list_item(node, false, true)?;
+        let mut items = list
+            .statements()
+            .iter()
+            .filter(|item| item != &&StatementListItem::Statement(Statement::Empty))
+            .peekable();
+
+        while let Some(item) = items.next() {
+            if items.peek().is_some() {
+                self.compile_stmt_list_item(item, false, true)?;
+            } else {
+                self.compile_stmt_list_item(item, use_expr, true)?;
             }
-            self.compile_stmt_list_item(last, use_expr, true)?;
         }
 
         let (num_bindings, compile_environment) = self.context.pop_compile_time_environment();

boa_parser/src/parser/statement/declaration/lexical.rs

@@ -69,11 +69,13 @@ where
         let _timer = Profiler::global().start_event("LexicalDeclaration", "Parsing");
         let tok = cursor.next(interner).or_abrupt()?;
 
-        match tok.kind() {
-            TokenKind::Keyword((Keyword::Const | Keyword::Let, true)) => Err(Error::general(
-                "Keyword must not contain escaped characters",
-                tok.span().start(),
-            )),
+        let lexical_declaration = match tok.kind() {
+            TokenKind::Keyword((Keyword::Const | Keyword::Let, true)) => {
+                return Err(Error::general(
+                    "Keyword must not contain escaped characters",
+                    tok.span().start(),
+                ))
+            }
             TokenKind::Keyword((Keyword::Const, false)) => BindingList::new(
                 self.allow_in,
                 self.allow_yield,
@@ -81,7 +83,7 @@ where
                 true,
                 self.loop_init,
             )
-            .parse(cursor, interner),
+            .parse(cursor, interner)?,
             TokenKind::Keyword((Keyword::Let, false)) => BindingList::new(
                 self.allow_in,
                 self.allow_yield,
@@ -89,9 +91,15 @@ where
                 false,
                 self.loop_init,
             )
-            .parse(cursor, interner),
+            .parse(cursor, interner)?,
             _ => unreachable!("unknown token found: {:?}", tok),
+        };
+
+        if !self.loop_init {
+            cursor.expect_semicolon("lexical declaration", interner)?;
         }
+
+        Ok(lexical_declaration)
     }
 }
 

boa_parser/src/parser/statement/mod.rs

@@ -313,9 +313,6 @@ where
             }
 
             items.push(item);
-
-            // move the cursor forward for any consecutive semicolon.
-            while cursor.next_if(Punctuator::Semicolon, interner)?.is_some() {}
         }
 
         items.sort_by(ast::StatementListItem::hoistable_order);

boa_parser/src/parser/tests/mod.rs

@@ -511,6 +511,7 @@ fn empty_statement() {
             if(a) ;
         ",
         vec![
+            Statement::Empty.into(),
             Statement::Empty.into(),
             Statement::Var(VarDeclaration(
                 vec![Variable::from_identifier(
@@ -527,6 +528,39 @@ fn empty_statement() {
     );
 }
 
+#[test]
+fn empty_statement_ends_directive_prologues() {
+    let interner = &mut Interner::default();
+    let a = interner.get_or_intern_static("a", utf16!("a"));
+    let use_strict = interner.get_or_intern_static("use strict", utf16!("use strict"));
+    let public = interner
+        .get_or_intern_static("public", utf16!("public"))
+        .into();
+    check_parser(
+        r#"
+            "a";
+            ;
+            "use strict";
+            let public = 5;
+        "#,
+        vec![
+            Statement::Expression(Expression::from(Literal::String(a))).into(),
+            Statement::Empty.into(),
+            Statement::Expression(Expression::from(Literal::String(use_strict))).into(),
+            Declaration::Lexical(LexicalDeclaration::Let(
+                vec![Variable::from_identifier(
+                    public,
+                    Some(Literal::from(5).into()),
+                )]
+                .try_into()
+                .unwrap(),
+            ))
+            .into(),
+        ],
+        interner,
+    );
+}
+
 #[test]
 fn hashbang_use_strict_no_with() {
     check_parser(