Refactor `IdentifierReference` parsing (#2055)

This Pull Request changes the following:

- Crate dedicated `IdentifierReference` parser and refactor `PrimaryExpression` and `PropertyDefinition` parsers to use it.
- Move `BindingIdentifier` and `LabelIdentifier` parsers from statement parser module to expression parser module to conform with the spec.
- Add and early error case while converting an `ObjectLiteral` to an `ObjectAssignmentPattern`

boa_engine/src/syntax/ast/node/operator/assign/mod.rs

@@ -8,7 +8,7 @@ use crate::syntax::ast::node::{
     ArrayDecl, DeclarationPattern, GetConstField, GetField, Identifier, Node, Object,
 };
 use boa_gc::{Finalize, Trace};
-use boa_interner::{Interner, ToInternedString};
+use boa_interner::{Interner, Sym, ToInternedString};
 
 #[cfg(feature = "deser")]
 use serde::{Deserialize, Serialize};
@@ -90,18 +90,18 @@ pub enum AssignTarget {
 impl AssignTarget {
     /// Converts the left-hand-side node of an assignment expression into it's an [`AssignTarget`].
     /// Returns `None` if the given node is an invalid left-hand-side for a assignment expression.
-    pub(crate) fn from_node(node: &Node) -> Option<Self> {
+    pub(crate) fn from_node(node: &Node, strict: bool) -> Option<Self> {
         match node {
             Node::Identifier(target) => Some(Self::Identifier(*target)),
             Node::GetPrivateField(target) => Some(Self::GetPrivateField(target.clone())),
             Node::GetConstField(target) => Some(Self::GetConstField(target.clone())),
             Node::GetField(target) => Some(Self::GetField(target.clone())),
             Node::Object(object) => {
-                let pattern = object_decl_to_declaration_pattern(object)?;
+                let pattern = object_decl_to_declaration_pattern(object, strict)?;
                 Some(Self::DeclarationPattern(pattern))
             }
             Node::ArrayDecl(array) => {
-                let pattern = array_decl_to_declaration_pattern(array)?;
+                let pattern = array_decl_to_declaration_pattern(array, strict)?;
                 Some(Self::DeclarationPattern(pattern))
             }
             _ => None,
@@ -140,11 +140,17 @@ impl From<GetField> for AssignTarget {
 }
 
 /// Converts an object literal into an object declaration pattern.
-pub(crate) fn object_decl_to_declaration_pattern(object: &Object) -> Option<DeclarationPattern> {
+pub(crate) fn object_decl_to_declaration_pattern(
+    object: &Object,
+    strict: bool,
+) -> Option<DeclarationPattern> {
     let mut bindings = Vec::new();
     let mut excluded_keys = Vec::new();
     for (i, property) in object.properties().iter().enumerate() {
         match property {
+            PropertyDefinition::IdentifierReference(ident) if strict && *ident == Sym::EVAL => {
+                return None
+            }
             PropertyDefinition::IdentifierReference(ident) => {
                 excluded_keys.push(*ident);
                 bindings.push(BindingPatternTypeObject::SingleName {
@@ -155,6 +161,9 @@ pub(crate) fn object_decl_to_declaration_pattern(object: &Object) -> Option<Decl
             }
             PropertyDefinition::Property(name, node) => match (name, node) {
                 (PropertyName::Literal(name), Node::Identifier(ident)) if *name == ident.sym() => {
+                    if strict && *name == Sym::EVAL {
+                        return None;
+                    }
                     excluded_keys.push(*name);
                     bindings.push(BindingPatternTypeObject::SingleName {
                         ident: *name,
@@ -196,7 +205,10 @@ pub(crate) fn object_decl_to_declaration_pattern(object: &Object) -> Option<Decl
 }
 
 /// Converts an array declaration into an array declaration pattern.
-pub(crate) fn array_decl_to_declaration_pattern(array: &ArrayDecl) -> Option<DeclarationPattern> {
+pub(crate) fn array_decl_to_declaration_pattern(
+    array: &ArrayDecl,
+    strict: bool,
+) -> Option<DeclarationPattern> {
     if array.has_trailing_comma_spread() {
         return None;
     }
@@ -227,11 +239,11 @@ pub(crate) fn array_decl_to_declaration_pattern(array: &ArrayDecl) -> Option<Dec
                         });
                     }
                     Node::ArrayDecl(array) => {
-                        let pattern = array_decl_to_declaration_pattern(array)?;
+                        let pattern = array_decl_to_declaration_pattern(array, strict)?;
                         bindings.push(BindingPatternTypeArray::BindingPatternRest { pattern });
                     }
                     Node::Object(object) => {
-                        let pattern = object_decl_to_declaration_pattern(object)?;
+                        let pattern = object_decl_to_declaration_pattern(object, strict)?;
                         bindings.push(BindingPatternTypeArray::BindingPatternRest { pattern });
                     }
                     _ => return None,
@@ -268,11 +280,11 @@ pub(crate) fn array_decl_to_declaration_pattern(array: &ArrayDecl) -> Option<Dec
                 AssignTarget::GetPrivateField(_) => return None,
             },
             Node::ArrayDecl(array) => {
-                let pattern = array_decl_to_declaration_pattern(array)?;
+                let pattern = array_decl_to_declaration_pattern(array, strict)?;
                 bindings.push(BindingPatternTypeArray::BindingPattern { pattern });
             }
             Node::Object(object) => {
-                let pattern = object_decl_to_declaration_pattern(object)?;
+                let pattern = object_decl_to_declaration_pattern(object, strict)?;
                 bindings.push(BindingPatternTypeArray::BindingPattern { pattern });
             }
             Node::GetField(get_field) => {

boa_engine/src/syntax/parser/expression/assignment/arrow_function.rs

@@ -19,8 +19,8 @@ use crate::syntax::{
     lexer::{Error as LexError, TokenKind},
     parser::{
         error::{ErrorContext, ParseError, ParseResult},
+        expression::BindingIdentifier,
         function::{FormalParameters, FunctionBody},
-        statement::BindingIdentifier,
         AllowAwait, AllowIn, AllowYield, Cursor, TokenParser,
     },
 };

boa_engine/src/syntax/parser/expression/assignment/mod.rs

@@ -229,7 +229,7 @@ where
                     }
 
                     cursor.next(interner)?.expect("= token vanished");
-                    if let Some(target) = AssignTarget::from_node(&lhs) {
+                    if let Some(target) = AssignTarget::from_node(&lhs, cursor.strict_mode()) {
                         if let AssignTarget::Identifier(ident) = target {
                             self.name = Some(ident.sym());
                         }

boa_engine/src/syntax/parser/expression/identifiers.rs

@@ -0,0 +1,238 @@
+//! Identifiers parsing.
+//!
+//! More information:
+//!  - [ECMAScript specification][spec]
+//!
+//! [spec]: https://tc39.es/ecma262/#sec-identifiers
+//!
+
+use crate::syntax::{
+    ast::{node::Identifier, Keyword},
+    lexer::{Error as LexError, TokenKind},
+    parser::{cursor::Cursor, AllowAwait, AllowYield, ParseError, TokenParser},
+};
+use boa_interner::{Interner, Sym};
+use boa_profiler::Profiler;
+use std::io::Read;
+
+const RESERVED_IDENTIFIERS_STRICT: [Sym; 9] = [
+    Sym::IMPLEMENTS,
+    Sym::INTERFACE,
+    Sym::LET,
+    Sym::PACKAGE,
+    Sym::PRIVATE,
+    Sym::PROTECTED,
+    Sym::PUBLIC,
+    Sym::STATIC,
+    Sym::YIELD,
+];
+
+/// Identifier reference parsing.
+///
+/// More information:
+///  - [ECMAScript specification][spec]
+///
+/// [spec]: https://tc39.es/ecma262/#prod-IdentifierReference
+#[derive(Debug, Clone, Copy)]
+pub(in crate::syntax::parser) struct IdentifierReference {
+    allow_yield: AllowYield,
+    allow_await: AllowAwait,
+}
+
+impl IdentifierReference {
+    /// Creates a new `IdentifierReference` parser.
+    pub(in crate::syntax::parser) fn new<Y, A>(allow_yield: Y, allow_await: A) -> Self
+    where
+        Y: Into<AllowYield>,
+        A: Into<AllowAwait>,
+    {
+        Self {
+            allow_yield: allow_yield.into(),
+            allow_await: allow_await.into(),
+        }
+    }
+}
+
+impl<R> TokenParser<R> for IdentifierReference
+where
+    R: Read,
+{
+    type Output = Identifier;
+
+    fn parse(
+        self,
+        cursor: &mut Cursor<R>,
+        interner: &mut Interner,
+    ) -> Result<Self::Output, ParseError> {
+        let _timer = Profiler::global().start_event("IdentifierReference", "Parsing");
+
+        let token = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?;
+
+        match token.kind() {
+            TokenKind::Identifier(ident)
+                if cursor.strict_mode() && RESERVED_IDENTIFIERS_STRICT.contains(ident) =>
+            {
+                Err(ParseError::general(
+                    "using future reserved keyword not allowed in strict mode IdentifierReference",
+                    token.span().start(),
+                ))
+            }
+            TokenKind::Identifier(ident) => Ok(Identifier::new(*ident)),
+            TokenKind::Keyword((Keyword::Let, _)) if cursor.strict_mode() => {
+                Err(ParseError::general(
+                    "using future reserved keyword not allowed in strict mode IdentifierReference",
+                    token.span().start(),
+                ))
+            }
+            TokenKind::Keyword((Keyword::Let, _)) => Ok(Identifier::new(Sym::LET)),
+            TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => {
+                // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield".
+                Err(ParseError::general(
+                    "Unexpected identifier",
+                    token.span().start(),
+                ))
+            }
+            TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => {
+                if cursor.strict_mode() {
+                    return Err(ParseError::general(
+                        "Unexpected strict mode reserved word",
+                        token.span().start(),
+                    ));
+                }
+                Ok(Identifier::new(Sym::YIELD))
+            }
+            TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => {
+                // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await".
+                Err(ParseError::general(
+                    "Unexpected identifier",
+                    token.span().start(),
+                ))
+            }
+            TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => {
+                Ok(Identifier::new(Sym::AWAIT))
+            }
+            _ => Err(ParseError::unexpected(
+                token.to_string(interner),
+                token.span(),
+                "IdentifierReference",
+            )),
+        }
+    }
+}
+
+/// Binding identifier parsing.
+///
+/// More information:
+///  - [ECMAScript specification][spec]
+///
+/// [spec]: https://tc39.es/ecma262/#prod-BindingIdentifier
+#[derive(Debug, Clone, Copy)]
+pub(in crate::syntax::parser) struct BindingIdentifier {
+    allow_yield: AllowYield,
+    allow_await: AllowAwait,
+}
+
+impl BindingIdentifier {
+    /// Creates a new `BindingIdentifier` parser.
+    pub(in crate::syntax::parser) fn new<Y, A>(allow_yield: Y, allow_await: A) -> Self
+    where
+        Y: Into<AllowYield>,
+        A: Into<AllowAwait>,
+    {
+        Self {
+            allow_yield: allow_yield.into(),
+            allow_await: allow_await.into(),
+        }
+    }
+}
+
+impl<R> TokenParser<R> for BindingIdentifier
+where
+    R: Read,
+{
+    type Output = Sym;
+
+    /// Strict mode parsing as per <https://tc39.es/ecma262/#sec-identifiers-static-semantics-early-errors>.
+    fn parse(
+        self,
+        cursor: &mut Cursor<R>,
+        interner: &mut Interner,
+    ) -> Result<Self::Output, ParseError> {
+        let _timer = Profiler::global().start_event("BindingIdentifier", "Parsing");
+
+        let next_token = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?;
+
+        match next_token.kind() {
+            TokenKind::Identifier(Sym::ARGUMENTS) if cursor.strict_mode() => {
+                Err(ParseError::lex(LexError::Syntax(
+                    "unexpected identifier 'arguments' in strict mode".into(),
+                    next_token.span().start(),
+                )))
+            }
+            TokenKind::Identifier(Sym::EVAL) if cursor.strict_mode() => {
+                Err(ParseError::lex(LexError::Syntax(
+                    "unexpected identifier 'eval' in strict mode".into(),
+                    next_token.span().start(),
+                )))
+            }
+            TokenKind::Identifier(ident) => {
+                if cursor.strict_mode() && RESERVED_IDENTIFIERS_STRICT.contains(ident) {
+                    return Err(ParseError::general(
+                        "using future reserved keyword not allowed in strict mode",
+                        next_token.span().start(),
+                    ));
+                }
+                Ok(*ident)
+            }
+            TokenKind::Keyword((Keyword::Let, _)) if cursor.strict_mode() => {
+                Err(ParseError::lex(LexError::Syntax(
+                    "unexpected identifier 'let' in strict mode".into(),
+                    next_token.span().start(),
+                )))
+            }
+            TokenKind::Keyword((Keyword::Let, _)) => Ok(Sym::LET),
+            TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => {
+                // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield".
+                Err(ParseError::general(
+                    "Unexpected identifier",
+                    next_token.span().start(),
+                ))
+            }
+            TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => {
+                if cursor.strict_mode() {
+                    Err(ParseError::general(
+                        "yield keyword in binding identifier not allowed in strict mode",
+                        next_token.span().start(),
+                    ))
+                } else {
+                    Ok(Sym::YIELD)
+                }
+            }
+            TokenKind::Keyword((Keyword::Await, _)) if cursor.arrow() => Ok(Sym::AWAIT),
+            TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => {
+                // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await".
+                Err(ParseError::general(
+                    "Unexpected identifier",
+                    next_token.span().start(),
+                ))
+            }
+            TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => Ok(Sym::AWAIT),
+            _ => Err(ParseError::expected(
+                ["identifier".to_owned()],
+                next_token.to_string(interner),
+                next_token.span(),
+                "binding identifier",
+            )),
+        }
+    }
+}
+
+/// Label identifier parsing.
+///
+/// This seems to be the same as a `BindingIdentifier`.
+///
+/// More information:
+///  - [ECMAScript specification][spec]
+///
+/// [spec]: https://tc39.es/ecma262/#prod-LabelIdentifier
+pub(in crate::syntax::parser) type LabelIdentifier = BindingIdentifier;

boa_engine/src/syntax/parser/expression/mod.rs

@@ -8,15 +8,17 @@
 //! [spec]: https://tc39.es/ecma262/#sec-ecmascript-language-expressions
 
 mod assignment;
+mod identifiers;
 mod left_hand_side;
 mod primary;
-#[cfg(test)]
-mod tests;
 mod unary;
 mod update;
 
-use self::assignment::ExponentiationExpression;
+pub(in crate::syntax::parser) mod await_expr;
-use super::{AllowAwait, AllowIn, AllowYield, Cursor, ParseResult, TokenParser};
+
+#[cfg(test)]
+mod tests;
+
 use crate::syntax::{
     ast::op::LogOp,
     ast::{
@@ -24,15 +26,18 @@ use crate::syntax::{
         Keyword, Punctuator,
     },
     lexer::{InputElement, TokenKind},
-    parser::ParseError,
+    parser::{
+        expression::assignment::ExponentiationExpression, AllowAwait, AllowIn, AllowYield, Cursor,
+        ParseError, ParseResult, TokenParser,
+    },
 };
 use boa_interner::{Interner, Sym};
 use boa_profiler::Profiler;
 use std::io::Read;
 
 pub(super) use self::{assignment::AssignmentExpression, primary::Initializer};
-pub(in crate::syntax::parser) mod await_expr;
 pub(in crate::syntax::parser) use {
+    identifiers::{BindingIdentifier, LabelIdentifier},
     left_hand_side::LeftHandSideExpression,
     primary::object_initializer::{
         AsyncGeneratorMethod, AsyncMethod, GeneratorMethod, PropertyName,

boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs

@@ -5,8 +5,8 @@ use crate::syntax::{
     ast::{node::AsyncFunctionExpr, Keyword, Position, Punctuator},
     lexer::{Error as LexError, TokenKind},
     parser::{
+        expression::BindingIdentifier,
         function::{FormalParameters, FunctionBody},
-        statement::BindingIdentifier,
         AllowYield, Cursor, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs

@@ -14,8 +14,8 @@ use crate::syntax::{
     ast::{node::AsyncGeneratorExpr, Keyword, Position, Punctuator},
     lexer::{Error as LexError, TokenKind},
     parser::{
+        expression::BindingIdentifier,
         function::{FormalParameters, FunctionBody},
-        statement::BindingIdentifier,
         Cursor, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs

@@ -2,8 +2,8 @@ use crate::syntax::{
     ast::{Keyword, Node},
     lexer::TokenKind,
     parser::{
-        statement::{BindingIdentifier, ClassTail},
+        expression::BindingIdentifier, statement::ClassTail, AllowAwait, AllowYield, Cursor,
-        AllowAwait, AllowYield, Cursor, ParseError, TokenParser,
+        ParseError, TokenParser,
     },
 };
 use boa_interner::{Interner, Sym};

boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs

@@ -14,8 +14,8 @@ use crate::syntax::{
     ast::{node::FunctionExpr, Keyword, Position, Punctuator},
     lexer::{Error as LexError, TokenKind},
     parser::{
+        expression::BindingIdentifier,
         function::{FormalParameters, FunctionBody},
-        statement::BindingIdentifier,
         Cursor, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/expression/primary/generator_expression/mod.rs

@@ -14,8 +14,8 @@ use crate::syntax::{
     ast::{node::GeneratorExpr, Position, Punctuator},
     lexer::{Error as LexError, TokenKind},
     parser::{
+        expression::BindingIdentifier,
         function::{FormalParameters, FunctionBody},
-        statement::BindingIdentifier,
         Cursor, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/expression/primary/mod.rs

@@ -33,7 +33,9 @@ use crate::syntax::{
     },
     lexer::{token::Numeric, InputElement, TokenKind},
     parser::{
-        expression::{primary::template::TemplateLiteral, Expression},
+        expression::{
+            identifiers::IdentifierReference, primary::template::TemplateLiteral, Expression,
+        },
         AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser,
     },
 };
@@ -85,15 +87,19 @@ where
 
         // TODO: tok currently consumes the token instead of peeking, so the token
         // isn't passed and consumed by parsers according to spec (EX: GeneratorExpression)
-        let tok = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?;
+        let tok = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?;
 
         match tok.kind() {
             TokenKind::Keyword((Keyword::This | Keyword::Async, true)) => Err(ParseError::general(
                 "Keyword must not contain escaped characters",
                 tok.span().start(),
             )),
-            TokenKind::Keyword((Keyword::This, false)) => Ok(Node::This),
+            TokenKind::Keyword((Keyword::This, false)) => {
+                cursor.next(interner).expect("token disappeared");
+                Ok(Node::This)
+            }
             TokenKind::Keyword((Keyword::Function, _)) => {
+                cursor.next(interner).expect("token disappeared");
                 let next_token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?;
                 if next_token.kind() == &TokenKind::Punctuator(Punctuator::Mul) {
                     GeneratorExpression::new(self.name)
@@ -106,9 +112,11 @@ where
                 }
             }
             TokenKind::Keyword((Keyword::Class, _)) => {
+                cursor.next(interner).expect("token disappeared");
                 ClassExpression::new(self.name, false, false).parse(cursor, interner)
             }
             TokenKind::Keyword((Keyword::Async, false)) => {
+                cursor.next(interner).expect("token disappeared");
                 let mul_peek = cursor.peek(1, interner)?.ok_or(ParseError::AbruptEnd)?;
                 if mul_peek.kind() == &TokenKind::Punctuator(Punctuator::Mul) {
                     AsyncGeneratorExpression::new(self.name)
@@ -121,6 +129,7 @@ where
                 }
             }
             TokenKind::Punctuator(Punctuator::OpenParen) => {
+                cursor.next(interner).expect("token disappeared");
                 cursor.set_goal(InputElement::RegExp);
                 let expr = Expression::new(self.name, true, self.allow_yield, self.allow_await)
                     .parse(cursor, interner)?;
@@ -128,71 +137,76 @@ where
                 Ok(expr)
             }
             TokenKind::Punctuator(Punctuator::OpenBracket) => {
+                cursor.next(interner).expect("token disappeared");
                 cursor.set_goal(InputElement::RegExp);
                 ArrayLiteral::new(self.allow_yield, self.allow_await)
                     .parse(cursor, interner)
                     .map(Node::ArrayDecl)
             }
             TokenKind::Punctuator(Punctuator::OpenBlock) => {
+                cursor.next(interner).expect("token disappeared");
                 cursor.set_goal(InputElement::RegExp);
                 Ok(ObjectLiteral::new(self.allow_yield, self.allow_await)
                     .parse(cursor, interner)?
                     .into())
             }
-            TokenKind::BooleanLiteral(boolean) => Ok(Const::from(*boolean).into()),
+            TokenKind::BooleanLiteral(boolean) => {
-            TokenKind::NullLiteral => Ok(Const::Null.into()),
+                let node = Const::from(*boolean).into();
-            TokenKind::Identifier(ident) => Ok(Identifier::new(*ident).into()),
+                cursor.next(interner).expect("token disappeared");
-            TokenKind::Keyword((Keyword::Let, _)) => Ok(Identifier::new(Sym::LET).into()),
+                Ok(node)
-            TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => {
+            }
-                // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield".
+            TokenKind::NullLiteral => {
-                Err(ParseError::general(
+                cursor.next(interner).expect("token disappeared");
-                    "Unexpected identifier",
+                Ok(Const::Null.into())
-                    tok.span().start(),
+            }
-                ))
+            TokenKind::Identifier(_)
-            }
+            | TokenKind::Keyword((Keyword::Let | Keyword::Yield | Keyword::Await, _)) => {
-            TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => {
+                IdentifierReference::new(self.allow_yield, self.allow_await)
-                if cursor.strict_mode() {
+                    .parse(cursor, interner)
-                    return Err(ParseError::general(
+                    .map(Node::from)
-                        "Unexpected strict mode reserved word",
+            }
-                        tok.span().start(),
+            TokenKind::StringLiteral(lit) => {
-                    ));
+                let node = Const::from(*lit).into();
-                }
+                cursor.next(interner).expect("token disappeared");
-                Ok(Identifier::new(Sym::YIELD).into())
+                Ok(node)
+            }
+            TokenKind::TemplateNoSubstitution(template_string) => {
+                let node = Const::from(
+                    template_string
+                        .to_owned_cooked(interner)
+                        .map_err(ParseError::lex)?,
+                )
+                .into();
+                cursor.next(interner).expect("token disappeared");
+                Ok(node)
+            }
+            TokenKind::NumericLiteral(Numeric::Integer(num)) => {
+                let node = Const::from(*num).into();
+                cursor.next(interner).expect("token disappeared");
+                Ok(node)
+            }
+            TokenKind::NumericLiteral(Numeric::Rational(num)) => {
+                let node = Const::from(*num).into();
+                cursor.next(interner).expect("token disappeared");
+                Ok(node)
+            }
+            TokenKind::NumericLiteral(Numeric::BigInt(num)) => {
+                let node = Const::from(num.clone()).into();
+                cursor.next(interner).expect("token disappeared");
+                Ok(node)
             }
-            TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => {
-                // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await".
-                Err(ParseError::general(
-                    "Unexpected identifier",
-                    tok.span().start(),
-                ))
-            }
-            TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => {
-                if cursor.strict_mode() {
-                    return Err(ParseError::general(
-                        "Unexpected strict mode reserved word",
-                        tok.span().start(),
-                    ));
-                }
-                Ok(Identifier::new(Sym::AWAIT).into())
-            }
-            TokenKind::StringLiteral(lit) => Ok(Const::from(*lit).into()),
-            TokenKind::TemplateNoSubstitution(template_string) => Ok(Const::from(
-                template_string
-                    .to_owned_cooked(interner)
-                    .map_err(ParseError::lex)?,
-            )
-            .into()),
-            TokenKind::NumericLiteral(Numeric::Integer(num)) => Ok(Const::from(*num).into()),
-            TokenKind::NumericLiteral(Numeric::Rational(num)) => Ok(Const::from(*num).into()),
-            TokenKind::NumericLiteral(Numeric::BigInt(num)) => Ok(Const::from(num.clone()).into()),
             TokenKind::RegularExpressionLiteral(body, flags) => {
-                Ok(Node::from(New::from(Call::new(
+                let node = Node::from(New::from(Call::new(
                     Identifier::new(Sym::REGEXP),
                     vec![Const::from(*body).into(), Const::from(*flags).into()],
-                ))))
+                )));
+                cursor.next(interner).expect("token disappeared");
+                Ok(node)
             }
             TokenKind::Punctuator(Punctuator::Div) => {
-                let tok = cursor.lex_regex(tok.span().start(), interner)?;
+                let position = tok.span().start();
+                cursor.next(interner).expect("token disappeared");
+                let tok = cursor.lex_regex(position, interner)?;
 
                 if let TokenKind::RegularExpressionLiteral(body, flags) = *tok.kind() {
                     Ok(Node::from(New::from(Call::new(
@@ -208,16 +222,18 @@ where
                     ))
                 }
             }
-            TokenKind::TemplateMiddle(template_string) => TemplateLiteral::new(
+            TokenKind::TemplateMiddle(template_string) => {
-                self.allow_yield,
+                let parser = TemplateLiteral::new(
-                self.allow_await,
+                    self.allow_yield,
-                tok.span().start(),
+                    self.allow_await,
-                template_string
+                    tok.span().start(),
-                    .to_owned_cooked(interner)
+                    template_string
-                    .map_err(ParseError::lex)?,
+                        .to_owned_cooked(interner)
-            )
+                        .map_err(ParseError::lex)?,
-            .parse(cursor, interner)
+                );
-            .map(Node::TemplateLit),
+                cursor.next(interner).expect("token disappeared");
+                parser.parse(cursor, interner).map(Node::TemplateLit)
+            }
             _ => Err(ParseError::unexpected(
                 tok.to_string(interner),
                 tok.span(),

boa_engine/src/syntax/parser/expression/primary/object_initializer/mod.rs

@@ -15,13 +15,13 @@ use crate::syntax::{
         node::{
             object::{self, MethodDefinition},
             AsyncFunctionExpr, AsyncGeneratorExpr, FormalParameterList, FunctionExpr,
-            GeneratorExpr, Identifier, Node, Object,
+            GeneratorExpr, Node, Object,
         },
         Const, Keyword, Punctuator,
     },
     lexer::{token::Numeric, Error as LexError, TokenKind},
     parser::{
-        expression::AssignmentExpression,
+        expression::{identifiers::IdentifierReference, AssignmentExpression},
         function::{FormalParameter, FormalParameters, FunctionBody, UniqueFormalParameters},
         AllowAwait, AllowIn, AllowYield, Cursor, ParseError, ParseResult, TokenParser,
     },
@@ -146,63 +146,8 @@ where
                 next_token.kind(),
                 TokenKind::Punctuator(Punctuator::CloseBlock | Punctuator::Comma)
             ) {
-                let token = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?;
+                let ident = IdentifierReference::new(self.allow_yield, self.allow_await)
-                let ident = match token.kind() {
+                    .parse(cursor, interner)?;
-                    TokenKind::Identifier(Sym::ARGUMENTS) if cursor.strict_mode() => {
-                        return Err(ParseError::lex(LexError::Syntax(
-                            "unexpected identifier 'arguments' in strict mode".into(),
-                            token.span().start(),
-                        )));
-                    }
-                    TokenKind::Identifier(Sym::EVAL) if cursor.strict_mode() => {
-                        return Err(ParseError::lex(LexError::Syntax(
-                            "unexpected identifier 'eval' in strict mode".into(),
-                            token.span().start(),
-                        )));
-                    }
-                    TokenKind::Identifier(ident) => Identifier::new(*ident),
-                    TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => {
-                        // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield".
-                        return Err(ParseError::general(
-                            "Unexpected identifier",
-                            token.span().start(),
-                        ));
-                    }
-                    TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => {
-                        if cursor.strict_mode() {
-                            // Early Error: It is a Syntax Error if the code matched by this production is contained in strict mode code.
-                            return Err(ParseError::general(
-                                "Unexpected strict mode reserved word",
-                                token.span().start(),
-                            ));
-                        }
-                        Identifier::new(Sym::YIELD)
-                    }
-                    TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => {
-                        // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await".
-                        return Err(ParseError::general(
-                            "Unexpected identifier",
-                            token.span().start(),
-                        ));
-                    }
-                    TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => {
-                        if cursor.strict_mode() {
-                            // Early Error: It is a Syntax Error if the code matched by this production is contained in strict mode code.
-                            return Err(ParseError::general(
-                                "Unexpected strict mode reserved word",
-                                token.span().start(),
-                            ));
-                        }
-                        Identifier::new(Sym::YIELD)
-                    }
-                    _ => {
-                        return Err(ParseError::unexpected(
-                            token.to_string(interner),
-                            token.span(),
-                            "expected IdentifierReference",
-                        ));
-                    }
-                };
                 return Ok(object::PropertyDefinition::property(ident.sym(), ident));
             }
         }

boa_engine/src/syntax/parser/function/mod.rs

@@ -18,8 +18,8 @@ use crate::syntax::{
     },
     lexer::{Error as LexError, InputElement, TokenKind},
     parser::{
-        expression::Initializer,
+        expression::{BindingIdentifier, Initializer},
-        statement::{ArrayBindingPattern, BindingIdentifier, ObjectBindingPattern, StatementList},
+        statement::{ArrayBindingPattern, ObjectBindingPattern, StatementList},
         AllowAwait, AllowYield, Cursor, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/mod.rs

@@ -1,25 +1,29 @@
 //! Boa parser implementation.
 
 mod cursor;
-pub mod error;
 mod expression;
-pub(crate) mod function;
 mod statement;
+
+pub(crate) mod function;
+
+pub mod error;
+
 #[cfg(test)]
 mod tests;
 
-pub use self::error::{ParseError, ParseResult};
-
-use self::cursor::Cursor;
 use crate::{
-    syntax::{ast::node::StatementList, lexer::TokenKind},
+    syntax::{
+        ast::{node::StatementList, Position},
+        lexer::TokenKind,
+        parser::cursor::Cursor,
+    },
     Context,
 };
 use boa_interner::{Interner, Sym};
 use rustc_hash::{FxHashMap, FxHashSet};
 use std::io::Read;
 
-use super::ast::Position;
+pub use self::error::{ParseError, ParseResult};
 
 /// Trait implemented by parsers.
 ///

boa_engine/src/syntax/parser/statement/break_stm/mod.rs

@@ -10,12 +10,12 @@
 #[cfg(test)]
 mod tests;
 
-use super::LabelIdentifier;
 use crate::syntax::{
     ast::{node::Break, Keyword, Punctuator},
     lexer::TokenKind,
     parser::{
         cursor::{Cursor, SemicolonResult},
+        expression::LabelIdentifier,
         AllowAwait, AllowYield, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/statement/continue_stm/mod.rs

@@ -15,7 +15,7 @@ use crate::syntax::{
     lexer::TokenKind,
     parser::{
         cursor::{Cursor, SemicolonResult},
-        statement::LabelIdentifier,
+        expression::LabelIdentifier,
         AllowAwait, AllowYield, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/statement/declaration/hoistable/class_decl/mod.rs

@@ -11,14 +11,14 @@ use crate::syntax::{
     lexer::{Error as LexError, TokenKind},
     parser::{
         expression::{
-            AssignmentExpression, AsyncGeneratorMethod, AsyncMethod, GeneratorMethod,
+            AssignmentExpression, AsyncGeneratorMethod, AsyncMethod, BindingIdentifier,
-            LeftHandSideExpression, PropertyName,
+            GeneratorMethod, LeftHandSideExpression, PropertyName,
         },
         function::{
             FormalParameter, FormalParameters, FunctionBody, UniqueFormalParameters,
             FUNCTION_BREAK_TOKENS,
         },
-        statement::{BindingIdentifier, StatementList},
+        statement::StatementList,
         AllowAwait, AllowDefault, AllowYield, Cursor, ParseError, TokenParser,
     },
 };

boa_engine/src/syntax/parser/statement/declaration/hoistable/mod.rs

@@ -24,8 +24,9 @@ use crate::syntax::{
     ast::{Keyword, Node, Position, Punctuator},
     lexer::TokenKind,
     parser::{
+        expression::BindingIdentifier,
         function::{FormalParameters, FunctionBody},
-        statement::{BindingIdentifier, LexError},
+        statement::LexError,
         AllowAwait, AllowDefault, AllowYield, Cursor, ParseError, ParseResult, TokenParser,
     },
 };

boa_engine/src/syntax/parser/statement/iteration/for_statement.rs

@@ -118,7 +118,8 @@ where
             }
             (Some(init), TokenKind::Keyword((Keyword::In, false))) => {
                 let init_position = token.span().start();
-                let init = node_to_iterable_loop_initializer(init, init_position)?;
+                let init =
+                    node_to_iterable_loop_initializer(init, init_position, cursor.strict_mode())?;
 
                 let _next = cursor.next(interner)?;
                 let expr = Expression::new(None, true, self.allow_yield, self.allow_await)
@@ -167,7 +168,8 @@ where
                 return Ok(ForInLoop::new(init, expr, body).into());
             }
             (Some(init), TokenKind::Keyword((Keyword::Of, false))) => {
-                let init = node_to_iterable_loop_initializer(init, init_position)?;
+                let init =
+                    node_to_iterable_loop_initializer(init, init_position, cursor.strict_mode())?;
 
                 let _next = cursor.next(interner)?;
                 let iterable = Expression::new(None, true, self.allow_yield, self.allow_await)
@@ -277,6 +279,7 @@ where
 fn node_to_iterable_loop_initializer(
     node: &Node,
     position: Position,
+    strict: bool,
 ) -> Result<IterableLoopInitializer, ParseError> {
     match node {
         Node::Identifier(name) => Ok(IterableLoopInitializer::Identifier(*name)),
@@ -333,7 +336,7 @@ fn node_to_iterable_loop_initializer(
             position,
         ))),
         Node::Object(object) => {
-            if let Some(pattern) = object_decl_to_declaration_pattern(object) {
+            if let Some(pattern) = object_decl_to_declaration_pattern(object, strict) {
                 Ok(IterableLoopInitializer::DeclarationPattern(pattern))
             } else {
                 Err(ParseError::lex(LexError::Syntax(
@@ -343,7 +346,7 @@ fn node_to_iterable_loop_initializer(
             }
         }
         Node::ArrayDecl(array) => {
-            if let Some(pattern) = array_decl_to_declaration_pattern(array) {
+            if let Some(pattern) = array_decl_to_declaration_pattern(array, strict) {
                 Ok(IterableLoopInitializer::DeclarationPattern(pattern))
             } else {
                 Err(ParseError::lex(LexError::Syntax(

boa_engine/src/syntax/parser/statement/labelled_stm/mod.rs

@@ -4,9 +4,9 @@ use crate::syntax::{
     parser::{
         cursor::Cursor,
         error::ParseError,
+        expression::LabelIdentifier,
         statement::{
-            declaration::hoistable::FunctionDeclaration, AllowAwait, AllowReturn, LabelIdentifier,
+            declaration::hoistable::FunctionDeclaration, AllowAwait, AllowReturn, Statement,
-            Statement,
         },
         AllowYield, TokenParser,
     },

boa_engine/src/syntax/parser/statement/mod.rs

@@ -52,9 +52,9 @@ use crate::syntax::{
         Keyword, Node, Punctuator,
     },
     lexer::{Error as LexError, InputElement, Token, TokenKind},
-    parser::expression::{await_expr::AwaitExpression, Initializer},
+    parser::expression::{await_expr::AwaitExpression, BindingIdentifier, Initializer},
 };
-use boa_interner::{Interner, Sym};
+use boa_interner::Interner;
 use boa_profiler::Profiler;
 use std::{io::Read, vec};
 
@@ -387,124 +387,6 @@ where
     }
 }
 
-/// Label identifier parsing.
-///
-/// This seems to be the same as a `BindingIdentifier`.
-///
-/// More information:
-///  - [ECMAScript specification][spec]
-///
-/// [spec]: https://tc39.es/ecma262/#prod-LabelIdentifier
-pub(super) type LabelIdentifier = BindingIdentifier;
-
-/// Binding identifier parsing.
-///
-/// More information:
-///  - [ECMAScript specification][spec]
-///
-/// [spec]: https://tc39.es/ecma262/#prod-BindingIdentifier
-#[derive(Debug, Clone, Copy)]
-pub(super) struct BindingIdentifier {
-    allow_yield: AllowYield,
-    allow_await: AllowAwait,
-}
-
-impl BindingIdentifier {
-    /// Creates a new `BindingIdentifier` parser.
-    pub(super) fn new<Y, A>(allow_yield: Y, allow_await: A) -> Self
-    where
-        Y: Into<AllowYield>,
-        A: Into<AllowAwait>,
-    {
-        Self {
-            allow_yield: allow_yield.into(),
-            allow_await: allow_await.into(),
-        }
-    }
-}
-
-impl<R> TokenParser<R> for BindingIdentifier
-where
-    R: Read,
-{
-    type Output = Sym;
-
-    /// Strict mode parsing as per <https://tc39.es/ecma262/#sec-identifiers-static-semantics-early-errors>.
-    fn parse(
-        self,
-        cursor: &mut Cursor<R>,
-        interner: &mut Interner,
-    ) -> Result<Self::Output, ParseError> {
-        let _timer = Profiler::global().start_event("BindingIdentifier", "Parsing");
-
-        let next_token = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?;
-
-        match next_token.kind() {
-            TokenKind::Identifier(Sym::ARGUMENTS) if cursor.strict_mode() => {
-                Err(ParseError::lex(LexError::Syntax(
-                    "unexpected identifier 'arguments' in strict mode".into(),
-                    next_token.span().start(),
-                )))
-            }
-            TokenKind::Identifier(Sym::EVAL) if cursor.strict_mode() => {
-                Err(ParseError::lex(LexError::Syntax(
-                    "unexpected identifier 'eval' in strict mode".into(),
-                    next_token.span().start(),
-                )))
-            }
-            TokenKind::Keyword((Keyword::Let, _)) if cursor.strict_mode() => {
-                Err(ParseError::lex(LexError::Syntax(
-                    "unexpected identifier 'let' in strict mode".into(),
-                    next_token.span().start(),
-                )))
-            }
-            TokenKind::Keyword((Keyword::Let, _)) => Ok(Sym::LET),
-            TokenKind::Identifier(ref s) => Ok(*s),
-            TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => {
-                // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield".
-                Err(ParseError::general(
-                    "Unexpected identifier",
-                    next_token.span().start(),
-                ))
-            }
-            TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => {
-                if cursor.strict_mode() {
-                    Err(ParseError::general(
-                        "yield keyword in binding identifier not allowed in strict mode",
-                        next_token.span().start(),
-                    ))
-                } else {
-                    Ok(Sym::YIELD)
-                }
-            }
-            TokenKind::Keyword((Keyword::Await, _)) if cursor.arrow() => Ok(Sym::AWAIT),
-            TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => {
-                // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await".
-                Err(ParseError::general(
-                    "Unexpected identifier",
-                    next_token.span().start(),
-                ))
-            }
-            TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => {
-                if cursor.strict_mode() {
-                    Err(ParseError::general(
-                        "await keyword in binding identifier not allowed in strict mode",
-                        next_token.span().start(),
-                    ))
-                } else {
-                    Ok(Sym::AWAIT)
-                }
-            }
-            _ => Err(ParseError::expected(
-                ["identifier".to_owned()],
-                next_token.to_string(interner),
-                next_token.span(),
-                "binding identifier",
-            )),
-        }
-    }
-}
-
 /// `ObjectBindingPattern` pattern parsing.
 ///
 /// More information: