From 2a63de3d6800c8343a7f21d4ce0a998ebe7ccf90 Mon Sep 17 00:00:00 2001 From: raskad <32105367+raskad@users.noreply.github.com> Date: Sat, 9 Apr 2022 17:52:39 +0000 Subject: [PATCH] Allow unicode escaped characters in identifiers that are keywords (#2021) This Pull Request changes the following: - Remove syntax error for unicode escaped characters in keywords from the lexer. - Adjust the lexer tokens for keywords to indicate if they contain unicode escaped characters. - Throw syntax errors in parser, when keywords cannot contain unicode escaped characters. --- boa_engine/src/syntax/lexer/identifier.rs | 9 +- boa_engine/src/syntax/lexer/tests.rs | 76 ++++++------- boa_engine/src/syntax/lexer/token.rs | 15 +-- .../expression/assignment/exponentiation.rs | 2 +- .../parser/expression/assignment/mod.rs | 4 +- .../parser/expression/assignment/yield.rs | 7 +- .../syntax/parser/expression/await_expr.rs | 2 +- .../parser/expression/left_hand_side/call.rs | 2 +- .../expression/left_hand_side/member.rs | 45 ++++---- .../src/syntax/parser/expression/mod.rs | 10 +- .../primary/async_function_expression/mod.rs | 6 +- .../primary/async_generator_expression/mod.rs | 6 +- .../primary/class_expression/mod.rs | 2 +- .../primary/function_expression/mod.rs | 2 +- .../syntax/parser/expression/primary/mod.rs | 20 ++-- .../primary/object_initializer/mod.rs | 47 +++++--- .../src/syntax/parser/expression/unary.rs | 9 +- .../syntax/parser/statement/break_stm/mod.rs | 2 +- .../parser/statement/continue_stm/mod.rs | 2 +- .../hoistable/async_function_decl/mod.rs | 12 +- .../hoistable/async_generator_decl/mod.rs | 12 +- .../declaration/hoistable/class_decl/mod.rs | 29 +++-- .../hoistable/function_decl/mod.rs | 2 +- .../hoistable/generator_decl/mod.rs | 6 +- .../statement/declaration/hoistable/mod.rs | 12 +- .../parser/statement/declaration/lexical.rs | 21 +++- .../parser/statement/declaration/mod.rs | 4 +- .../syntax/parser/statement/expression/mod.rs | 34 ++++-- .../src/syntax/parser/statement/if_stm/mod.rs | 105 ++++++++++-------- .../statement/iteration/do_while_statement.rs | 28 +++-- .../statement/iteration/for_statement.rs | 19 +++- .../statement/iteration/while_statement.rs | 2 +- .../parser/statement/labelled_stm/mod.rs | 4 +- boa_engine/src/syntax/parser/statement/mod.rs | 38 +++---- .../syntax/parser/statement/return_stm/mod.rs | 2 +- .../src/syntax/parser/statement/switch/mod.rs | 26 +++-- .../src/syntax/parser/statement/throw/mod.rs | 2 +- .../syntax/parser/statement/try_stm/catch.rs | 2 +- .../parser/statement/try_stm/finally.rs | 2 +- .../syntax/parser/statement/try_stm/mod.rs | 38 ++++--- .../syntax/parser/statement/variable/mod.rs | 2 +- 41 files changed, 403 insertions(+), 267 deletions(-) diff --git a/boa_engine/src/syntax/lexer/identifier.rs b/boa_engine/src/syntax/lexer/identifier.rs index c114f6ed60..330d73ec90 100644 --- a/boa_engine/src/syntax/lexer/identifier.rs +++ b/boa_engine/src/syntax/lexer/identifier.rs @@ -90,13 +90,6 @@ impl Tokenizer for Identifier { Self::take_identifier_name(cursor, start_pos, self.init)?; let token_kind = if let Ok(keyword) = identifier_name.parse() { - if contains_escaped_chars { - return Err(Error::Syntax( - "unicode escaped characters are not allowed in keyword".into(), - start_pos, - )); - } - if cursor.strict_mode() && keyword == Keyword::With { return Err(Error::Syntax( "using 'with' statement not allowed in strict mode".into(), @@ -108,7 +101,7 @@ impl Tokenizer for Identifier { Keyword::True => TokenKind::BooleanLiteral(true), Keyword::False => TokenKind::BooleanLiteral(false), Keyword::Null => TokenKind::NullLiteral, - _ => TokenKind::Keyword(keyword), + _ => TokenKind::Keyword((keyword, contains_escaped_chars)), } } else { if cursor.strict_mode() diff --git a/boa_engine/src/syntax/lexer/tests.rs b/boa_engine/src/syntax/lexer/tests.rs index a5e7a3e713..2553f25c2b 100644 --- a/boa_engine/src/syntax/lexer/tests.rs +++ b/boa_engine/src/syntax/lexer/tests.rs @@ -36,7 +36,7 @@ fn check_single_line_comment() { let mut interner = Interner::default(); let expected = [ - TokenKind::Keyword(Keyword::Var), + TokenKind::Keyword((Keyword::Var, false)), TokenKind::LineTerminator, TokenKind::LineTerminator, TokenKind::BooleanLiteral(true), @@ -52,7 +52,7 @@ fn check_single_line_comment_with_crlf_ending() { let mut interner = Interner::default(); let expected = [ - TokenKind::Keyword(Keyword::Var), + TokenKind::Keyword((Keyword::Var, false)), TokenKind::LineTerminator, TokenKind::LineTerminator, TokenKind::BooleanLiteral(true), @@ -69,7 +69,7 @@ fn check_multi_line_comment() { let sym = interner.get_or_intern_static("x"); let expected = [ - TokenKind::Keyword(Keyword::Var), + TokenKind::Keyword((Keyword::Var, false)), TokenKind::LineTerminator, TokenKind::identifier(sym), ]; @@ -251,40 +251,40 @@ fn check_keywords() { let mut interner = Interner::default(); let expected = [ - TokenKind::Keyword(Keyword::Await), - TokenKind::Keyword(Keyword::Break), - TokenKind::Keyword(Keyword::Case), - TokenKind::Keyword(Keyword::Catch), - TokenKind::Keyword(Keyword::Class), - TokenKind::Keyword(Keyword::Const), - TokenKind::Keyword(Keyword::Continue), - TokenKind::Keyword(Keyword::Debugger), - TokenKind::Keyword(Keyword::Default), - TokenKind::Keyword(Keyword::Delete), - TokenKind::Keyword(Keyword::Do), - TokenKind::Keyword(Keyword::Else), - TokenKind::Keyword(Keyword::Export), - TokenKind::Keyword(Keyword::Extends), - TokenKind::Keyword(Keyword::Finally), - TokenKind::Keyword(Keyword::For), - TokenKind::Keyword(Keyword::Function), - TokenKind::Keyword(Keyword::If), - TokenKind::Keyword(Keyword::Import), - TokenKind::Keyword(Keyword::In), - TokenKind::Keyword(Keyword::InstanceOf), - TokenKind::Keyword(Keyword::New), - TokenKind::Keyword(Keyword::Return), - TokenKind::Keyword(Keyword::Super), - TokenKind::Keyword(Keyword::Switch), - TokenKind::Keyword(Keyword::This), - TokenKind::Keyword(Keyword::Throw), - TokenKind::Keyword(Keyword::Try), - TokenKind::Keyword(Keyword::TypeOf), - TokenKind::Keyword(Keyword::Var), - TokenKind::Keyword(Keyword::Void), - TokenKind::Keyword(Keyword::While), - TokenKind::Keyword(Keyword::With), - TokenKind::Keyword(Keyword::Yield), + TokenKind::Keyword((Keyword::Await, false)), + TokenKind::Keyword((Keyword::Break, false)), + TokenKind::Keyword((Keyword::Case, false)), + TokenKind::Keyword((Keyword::Catch, false)), + TokenKind::Keyword((Keyword::Class, false)), + TokenKind::Keyword((Keyword::Const, false)), + TokenKind::Keyword((Keyword::Continue, false)), + TokenKind::Keyword((Keyword::Debugger, false)), + TokenKind::Keyword((Keyword::Default, false)), + TokenKind::Keyword((Keyword::Delete, false)), + TokenKind::Keyword((Keyword::Do, false)), + TokenKind::Keyword((Keyword::Else, false)), + TokenKind::Keyword((Keyword::Export, false)), + TokenKind::Keyword((Keyword::Extends, false)), + TokenKind::Keyword((Keyword::Finally, false)), + TokenKind::Keyword((Keyword::For, false)), + TokenKind::Keyword((Keyword::Function, false)), + TokenKind::Keyword((Keyword::If, false)), + TokenKind::Keyword((Keyword::Import, false)), + TokenKind::Keyword((Keyword::In, false)), + TokenKind::Keyword((Keyword::InstanceOf, false)), + TokenKind::Keyword((Keyword::New, false)), + TokenKind::Keyword((Keyword::Return, false)), + TokenKind::Keyword((Keyword::Super, false)), + TokenKind::Keyword((Keyword::Switch, false)), + TokenKind::Keyword((Keyword::This, false)), + TokenKind::Keyword((Keyword::Throw, false)), + TokenKind::Keyword((Keyword::Try, false)), + TokenKind::Keyword((Keyword::TypeOf, false)), + TokenKind::Keyword((Keyword::Var, false)), + TokenKind::Keyword((Keyword::Void, false)), + TokenKind::Keyword((Keyword::While, false)), + TokenKind::Keyword((Keyword::With, false)), + TokenKind::Keyword((Keyword::Yield, false)), ]; expect_tokens(&mut lexer, &expected, &mut interner); @@ -299,7 +299,7 @@ fn check_variable_definition_tokens() { let a_sym = interner.get_or_intern_static("a"); let hello_sym = interner.get_or_intern_static("hello"); let expected = [ - TokenKind::Keyword(Keyword::Let), + TokenKind::Keyword((Keyword::Let, false)), TokenKind::identifier(a_sym), TokenKind::Punctuator(Punctuator::Assign), TokenKind::string_literal(hello_sym), diff --git a/boa_engine/src/syntax/lexer/token.rs b/boa_engine/src/syntax/lexer/token.rs index c08f9667bb..b2aeb29141 100644 --- a/boa_engine/src/syntax/lexer/token.rs +++ b/boa_engine/src/syntax/lexer/token.rs @@ -105,8 +105,8 @@ pub enum TokenKind { /// A private identifier. PrivateIdentifier(Sym), - /// A keyword. - Keyword(Keyword), + /// A keyword and a flag if the keyword contains unicode escaped chars. + Keyword((Keyword, bool)), /// A `null` literal. NullLiteral, @@ -142,8 +142,8 @@ impl From for TokenKind { } } -impl From for TokenKind { - fn from(kw: Keyword) -> Self { +impl From<(Keyword, bool)> for TokenKind { + fn from(kw: (Keyword, bool)) -> Self { Self::Keyword(kw) } } @@ -176,11 +176,6 @@ impl TokenKind { Self::Identifier(ident) } - /// Creates a `Keyword` token kind. - pub fn keyword(keyword: Keyword) -> Self { - Self::Keyword(keyword) - } - /// Creates a `NumericLiteral` token kind. pub fn numeric_literal(lit: L) -> Self where @@ -229,7 +224,7 @@ impl TokenKind { Self::EOF => "end of file".to_owned(), Self::Identifier(ident) => interner.resolve_expect(ident).to_owned(), Self::PrivateIdentifier(ident) => format!("#{}", interner.resolve_expect(ident)), - Self::Keyword(word) => word.to_string(), + Self::Keyword((word, _)) => word.to_string(), Self::NullLiteral => "null".to_owned(), Self::NumericLiteral(Numeric::Rational(num)) => num.to_string(), Self::NumericLiteral(Numeric::Integer(num)) => num.to_string(), diff --git a/boa_engine/src/syntax/parser/expression/assignment/exponentiation.rs b/boa_engine/src/syntax/parser/expression/assignment/exponentiation.rs index d543074ede..ed7b6d97c3 100644 --- a/boa_engine/src/syntax/parser/expression/assignment/exponentiation.rs +++ b/boa_engine/src/syntax/parser/expression/assignment/exponentiation.rs @@ -70,7 +70,7 @@ where Ok(if let Some(tok) = cursor.peek(0, interner)? { matches!( tok.kind(), - TokenKind::Keyword(Keyword::Delete | Keyword::Void | Keyword::TypeOf) + TokenKind::Keyword((Keyword::Delete | Keyword::Void | Keyword::TypeOf, _)) | TokenKind::Punctuator( Punctuator::Add | Punctuator::Sub | Punctuator::Not | Punctuator::Neg ) diff --git a/boa_engine/src/syntax/parser/expression/assignment/mod.rs b/boa_engine/src/syntax/parser/expression/assignment/mod.rs index 4a82a93571..e949cb969c 100644 --- a/boa_engine/src/syntax/parser/expression/assignment/mod.rs +++ b/boa_engine/src/syntax/parser/expression/assignment/mod.rs @@ -97,12 +97,12 @@ where .kind() { // [+Yield]YieldExpression[?In, ?Await] - TokenKind::Keyword(Keyword::Yield) if self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => { return YieldExpression::new(self.allow_in, self.allow_await) .parse(cursor, interner) } // ArrowFunction[?In, ?Yield, ?Await] -> ArrowParameters[?Yield, ?Await] -> BindingIdentifier[?Yield, ?Await] - TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => { + TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => { if let Ok(tok) = cursor.peek_expect_no_lineterminator(1, "assignment expression", interner) { diff --git a/boa_engine/src/syntax/parser/expression/assignment/yield.rs b/boa_engine/src/syntax/parser/expression/assignment/yield.rs index c9b1b3dc77..b62bef9c04 100644 --- a/boa_engine/src/syntax/parser/expression/assignment/yield.rs +++ b/boa_engine/src/syntax/parser/expression/assignment/yield.rs @@ -58,7 +58,7 @@ where let _timer = Profiler::global().start_event("YieldExpression", "Parsing"); cursor.expect( - TokenKind::Keyword(Keyword::Yield), + TokenKind::Keyword((Keyword::Yield, false)), "yield expression", interner, )?; @@ -87,7 +87,7 @@ where | Punctuator::OpenBlock | Punctuator::Div, ) - | TokenKind::Keyword( + | TokenKind::Keyword(( Keyword::Yield | Keyword::Await | Keyword::Delete @@ -98,7 +98,8 @@ where | Keyword::Function | Keyword::Class | Keyword::Async, - ) + _, + )) | TokenKind::BooleanLiteral(_) | TokenKind::NullLiteral | TokenKind::StringLiteral(_) diff --git a/boa_engine/src/syntax/parser/expression/await_expr.rs b/boa_engine/src/syntax/parser/expression/await_expr.rs index fcfc91881e..3f11ca7a23 100644 --- a/boa_engine/src/syntax/parser/expression/await_expr.rs +++ b/boa_engine/src/syntax/parser/expression/await_expr.rs @@ -53,7 +53,7 @@ where interner: &mut Interner, ) -> Result { cursor.expect( - TokenKind::Keyword(Keyword::Await), + TokenKind::Keyword((Keyword::Await, false)), "Await expression parsing", interner, )?; diff --git a/boa_engine/src/syntax/parser/expression/left_hand_side/call.rs b/boa_engine/src/syntax/parser/expression/left_hand_side/call.rs index 9ca3775ca6..bfb2e74a94 100644 --- a/boa_engine/src/syntax/parser/expression/left_hand_side/call.rs +++ b/boa_engine/src/syntax/parser/expression/left_hand_side/call.rs @@ -94,7 +94,7 @@ where TokenKind::Identifier(name) => { lhs = GetConstField::new(lhs, *name).into(); } - TokenKind::Keyword(kw) => { + TokenKind::Keyword((kw, _)) => { lhs = GetConstField::new(lhs, kw.to_sym(interner)).into(); } _ => { diff --git a/boa_engine/src/syntax/parser/expression/left_hand_side/member.rs b/boa_engine/src/syntax/parser/expression/left_hand_side/member.rs index cd4cfde062..306ceee712 100644 --- a/boa_engine/src/syntax/parser/expression/left_hand_side/member.rs +++ b/boa_engine/src/syntax/parser/expression/left_hand_side/member.rs @@ -64,27 +64,32 @@ where fn parse(self, cursor: &mut Cursor, interner: &mut Interner) -> ParseResult { let _timer = Profiler::global().start_event("MemberExpression", "Parsing"); - let mut lhs = if cursor - .peek(0, interner)? - .ok_or(ParseError::AbruptEnd)? - .kind() - == &TokenKind::Keyword(Keyword::New) - { - let _next = cursor.next(interner).expect("new keyword disappeared"); - let lhs = self.parse(cursor, interner)?; - let args = match cursor.peek(0, interner)? { - Some(next) if next.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { - Arguments::new(self.allow_yield, self.allow_await).parse(cursor, interner)? - } - _ => Box::new([]), - }; - let call_node = Call::new(lhs, args); + let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; + let mut lhs = match token.kind() { + TokenKind::Keyword((Keyword::New, true)) => { + return Err(ParseError::general( + "keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::New, false)) => { + let _next = cursor.next(interner).expect("new keyword disappeared"); + let lhs = self.parse(cursor, interner)?; + let args = match cursor.peek(0, interner)? { + Some(next) if next.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => { + Arguments::new(self.allow_yield, self.allow_await) + .parse(cursor, interner)? + } + _ => Box::new([]), + }; + let call_node = Call::new(lhs, args); - Node::from(New::from(call_node)) - } else { - PrimaryExpression::new(self.name, self.allow_yield, self.allow_await) - .parse(cursor, interner)? + Node::from(New::from(call_node)) + } + _ => PrimaryExpression::new(self.name, self.allow_yield, self.allow_await) + .parse(cursor, interner)?, }; + while let Some(tok) = cursor.peek(0, interner)? { match tok.kind() { TokenKind::Punctuator(Punctuator::Dot) => { @@ -96,7 +101,7 @@ where match token.kind() { TokenKind::Identifier(name) => lhs = GetConstField::new(lhs, *name).into(), - TokenKind::Keyword(kw) => { + TokenKind::Keyword((kw, _)) => { lhs = GetConstField::new(lhs, kw.to_sym(interner)).into(); } TokenKind::BooleanLiteral(bool) => { diff --git a/boa_engine/src/syntax/parser/expression/mod.rs b/boa_engine/src/syntax/parser/expression/mod.rs index f9e87d94cc..3f84333e31 100644 --- a/boa_engine/src/syntax/parser/expression/mod.rs +++ b/boa_engine/src/syntax/parser/expression/mod.rs @@ -98,7 +98,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo $lower::new($( self.$low_param ),*).parse(cursor, interner)? ).into(); } - TokenKind::Keyword(op) if $( op == $op )||* => { + TokenKind::Keyword((op, false)) if $( op == $op )||* => { let _next = cursor.next(interner).expect("token disappeared"); lhs = BinOp::new( op.as_binop().expect("Could not get binary operation."), @@ -540,7 +540,13 @@ where ) .into(); } - TokenKind::Keyword(op) + TokenKind::Keyword((Keyword::InstanceOf | Keyword::In, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + tok.span().start(), + )); + } + TokenKind::Keyword((op, false)) if op == Keyword::InstanceOf || (op == Keyword::In && self.allow_in == AllowIn(true)) => { diff --git a/boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs index 566a103246..d70a4a3b14 100644 --- a/boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs +++ b/boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs @@ -55,7 +55,11 @@ where ) -> Result { let _timer = Profiler::global().start_event("AsyncFunctionExpression", "Parsing"); cursor.peek_expect_no_lineterminator(0, "async function expression", interner)?; - cursor.expect(Keyword::Function, "async function expression", interner)?; + cursor.expect( + (Keyword::Function, false), + "async function expression", + interner, + )?; let name = match cursor .peek(0, interner)? diff --git a/boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs index d92300755c..7f4da639a5 100644 --- a/boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs +++ b/boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs @@ -59,7 +59,11 @@ where let _timer = Profiler::global().start_event("AsyncGeneratorExpression", "Parsing"); cursor.peek_expect_no_lineterminator(0, "async generator expression", interner)?; - cursor.expect(Keyword::Function, "async generator expression", interner)?; + cursor.expect( + (Keyword::Function, false), + "async generator expression", + interner, + )?; cursor.expect( TokenKind::Punctuator(Punctuator::Mul), "async generator expression", diff --git a/boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs index d5395d2d15..1dbfd93926 100644 --- a/boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs +++ b/boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs @@ -56,7 +56,7 @@ where let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; let name = match token.kind() { - TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => { + TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => { BindingIdentifier::new(self.allow_yield, self.allow_await) .parse(cursor, interner)? } diff --git a/boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs index a0d8e497e1..a85b422bf1 100644 --- a/boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs +++ b/boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs @@ -64,7 +64,7 @@ where .ok_or(ParseError::AbruptEnd)? .kind() { - TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => { + TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => { Some(BindingIdentifier::new(false, false).parse(cursor, interner)?) } _ => self.name, diff --git a/boa_engine/src/syntax/parser/expression/primary/mod.rs b/boa_engine/src/syntax/parser/expression/primary/mod.rs index 6a0e2cad8a..efd69d4f69 100644 --- a/boa_engine/src/syntax/parser/expression/primary/mod.rs +++ b/boa_engine/src/syntax/parser/expression/primary/mod.rs @@ -88,8 +88,12 @@ where let tok = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { - TokenKind::Keyword(Keyword::This) => Ok(Node::This), - TokenKind::Keyword(Keyword::Function) => { + TokenKind::Keyword((Keyword::This | Keyword::Async, true)) => Err(ParseError::general( + "Keyword must not contain escaped characters", + tok.span().start(), + )), + TokenKind::Keyword((Keyword::This, false)) => Ok(Node::This), + TokenKind::Keyword((Keyword::Function, _)) => { let next_token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; if next_token.kind() == &TokenKind::Punctuator(Punctuator::Mul) { GeneratorExpression::new(self.name) @@ -101,10 +105,10 @@ where .map(Node::from) } } - TokenKind::Keyword(Keyword::Class) => { + TokenKind::Keyword((Keyword::Class, _)) => { ClassExpression::new(self.name, false, false).parse(cursor, interner) } - TokenKind::Keyword(Keyword::Async) => { + TokenKind::Keyword((Keyword::Async, false)) => { let mul_peek = cursor.peek(1, interner)?.ok_or(ParseError::AbruptEnd)?; if mul_peek.kind() == &TokenKind::Punctuator(Punctuator::Mul) { AsyncGeneratorExpression::new(self.name) @@ -138,14 +142,14 @@ where TokenKind::BooleanLiteral(boolean) => Ok(Const::from(*boolean).into()), TokenKind::NullLiteral => Ok(Const::Null.into()), TokenKind::Identifier(ident) => Ok(Identifier::new(*ident).into()), - TokenKind::Keyword(Keyword::Yield) if self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => { // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield". Err(ParseError::general( "Unexpected identifier", tok.span().start(), )) } - TokenKind::Keyword(Keyword::Yield) if !self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => { if cursor.strict_mode() { return Err(ParseError::general( "Unexpected strict mode reserved word", @@ -154,14 +158,14 @@ where } Ok(Identifier::new(Sym::YIELD).into()) } - TokenKind::Keyword(Keyword::Await) if self.allow_await.0 => { + TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => { // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await". Err(ParseError::general( "Unexpected identifier", tok.span().start(), )) } - TokenKind::Keyword(Keyword::Await) if !self.allow_await.0 => { + TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => { if cursor.strict_mode() { return Err(ParseError::general( "Unexpected strict mode reserved word", diff --git a/boa_engine/src/syntax/parser/expression/primary/object_initializer/mod.rs b/boa_engine/src/syntax/parser/expression/primary/object_initializer/mod.rs index 7245a9775a..42c3d6c261 100644 --- a/boa_engine/src/syntax/parser/expression/primary/object_initializer/mod.rs +++ b/boa_engine/src/syntax/parser/expression/primary/object_initializer/mod.rs @@ -161,14 +161,14 @@ where ))); } TokenKind::Identifier(ident) => Identifier::new(*ident), - TokenKind::Keyword(Keyword::Yield) if self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => { // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield". return Err(ParseError::general( "Unexpected identifier", token.span().start(), )); } - TokenKind::Keyword(Keyword::Yield) if !self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => { if cursor.strict_mode() { // Early Error: It is a Syntax Error if the code matched by this production is contained in strict mode code. return Err(ParseError::general( @@ -178,14 +178,14 @@ where } Identifier::new(Sym::YIELD) } - TokenKind::Keyword(Keyword::Await) if self.allow_await.0 => { + TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => { // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await". return Err(ParseError::general( "Unexpected identifier", token.span().start(), )); } - TokenKind::Keyword(Keyword::Await) if !self.allow_await.0 => { + TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => { if cursor.strict_mode() { // Early Error: It is a Syntax Error if the code matched by this production is contained in strict mode code. return Err(ParseError::general( @@ -215,25 +215,36 @@ where } //Async [AsyncMethod, AsyncGeneratorMethod] object methods - if cursor.next_if(Keyword::Async, interner)?.is_some() { - cursor.peek_expect_no_lineterminator(0, "Async object methods", interner)?; - - let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; - if let TokenKind::Punctuator(Punctuator::Mul) = token.kind() { + let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; + match token.kind() { + TokenKind::Keyword((Keyword::Async, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Async, false)) => { + cursor.next(interner)?.expect("token disappeared"); + cursor.peek_expect_no_lineterminator(0, "Async object methods", interner)?; + + let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; + if let TokenKind::Punctuator(Punctuator::Mul) = token.kind() { + let (property_name, method) = + AsyncGeneratorMethod::new(self.allow_yield, self.allow_await) + .parse(cursor, interner)?; + return Ok(object::PropertyDefinition::method_definition( + method, + property_name, + )); + } let (property_name, method) = - AsyncGeneratorMethod::new(self.allow_yield, self.allow_await) - .parse(cursor, interner)?; + AsyncMethod::new(self.allow_yield, self.allow_await).parse(cursor, interner)?; return Ok(object::PropertyDefinition::method_definition( method, property_name, )); } - let (property_name, method) = - AsyncMethod::new(self.allow_yield, self.allow_await).parse(cursor, interner)?; - return Ok(object::PropertyDefinition::method_definition( - method, - property_name, - )); + _ => {} } if cursor @@ -465,7 +476,7 @@ where Numeric::Integer(num) => Node::Const(Const::from(*num)).into(), Numeric::BigInt(num) => Node::Const(Const::from(num.clone())).into(), }, - TokenKind::Keyword(word) => { + TokenKind::Keyword((word, _)) => { Node::Const(Const::from(interner.get_or_intern_static(word.as_str()))).into() } TokenKind::NullLiteral => Node::Const(Const::from(Sym::NULL)).into(), diff --git a/boa_engine/src/syntax/parser/expression/unary.rs b/boa_engine/src/syntax/parser/expression/unary.rs index ba79c3260b..011c927715 100644 --- a/boa_engine/src/syntax/parser/expression/unary.rs +++ b/boa_engine/src/syntax/parser/expression/unary.rs @@ -66,7 +66,10 @@ where let tok = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; let token_start = tok.span().start(); match tok.kind() { - TokenKind::Keyword(Keyword::Delete) => { + TokenKind::Keyword((Keyword::Delete | Keyword::Void | Keyword::TypeOf, true)) => Err( + ParseError::general("Keyword must not contain escaped characters", token_start), + ), + TokenKind::Keyword((Keyword::Delete, false)) => { cursor.next(interner)?.expect("Delete keyword vanished"); let position = cursor .peek(0, interner)? @@ -93,11 +96,11 @@ where Ok(node::UnaryOp::new(UnaryOp::Delete, val).into()) } - TokenKind::Keyword(Keyword::Void) => { + TokenKind::Keyword((Keyword::Void, false)) => { cursor.next(interner)?.expect("Void keyword vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::Void, self.parse(cursor, interner)?).into()) } - TokenKind::Keyword(Keyword::TypeOf) => { + TokenKind::Keyword((Keyword::TypeOf, false)) => { cursor.next(interner)?.expect("TypeOf keyword vanished"); // Consume the token. Ok(node::UnaryOp::new(UnaryOp::TypeOf, self.parse(cursor, interner)?).into()) } diff --git a/boa_engine/src/syntax/parser/statement/break_stm/mod.rs b/boa_engine/src/syntax/parser/statement/break_stm/mod.rs index efdb435404..8c03b11060 100644 --- a/boa_engine/src/syntax/parser/statement/break_stm/mod.rs +++ b/boa_engine/src/syntax/parser/statement/break_stm/mod.rs @@ -63,7 +63,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("BreakStatement", "Parsing"); - cursor.expect(Keyword::Break, "break statement", interner)?; + cursor.expect((Keyword::Break, false), "break statement", interner)?; let label = if let SemicolonResult::Found(tok) = cursor.peek_semicolon(interner)? { match tok { diff --git a/boa_engine/src/syntax/parser/statement/continue_stm/mod.rs b/boa_engine/src/syntax/parser/statement/continue_stm/mod.rs index 5a8fecee72..b42a6a4755 100644 --- a/boa_engine/src/syntax/parser/statement/continue_stm/mod.rs +++ b/boa_engine/src/syntax/parser/statement/continue_stm/mod.rs @@ -63,7 +63,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("ContinueStatement", "Parsing"); - cursor.expect(Keyword::Continue, "continue statement", interner)?; + cursor.expect((Keyword::Continue, false), "continue statement", interner)?; let label = if let SemicolonResult::Found(tok) = cursor.peek_semicolon(interner)? { match tok { diff --git a/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_function_decl/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_function_decl/mod.rs index a068bae3f6..f01cd52373 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_function_decl/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_function_decl/mod.rs @@ -80,9 +80,17 @@ where cursor: &mut Cursor, interner: &mut Interner, ) -> Result { - cursor.expect(Keyword::Async, "async function declaration", interner)?; + cursor.expect( + (Keyword::Async, false), + "async function declaration", + interner, + )?; cursor.peek_expect_no_lineterminator(0, "async function declaration", interner)?; - cursor.expect(Keyword::Function, "async function declaration", interner)?; + cursor.expect( + (Keyword::Function, false), + "async function declaration", + interner, + )?; let result = parse_callable_declaration(&self, cursor, interner)?; diff --git a/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_generator_decl/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_generator_decl/mod.rs index 276e33d0b4..56f6cacf1a 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_generator_decl/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/hoistable/async_generator_decl/mod.rs @@ -98,9 +98,17 @@ where cursor: &mut Cursor, interner: &mut Interner, ) -> Result { - cursor.expect(Keyword::Async, "async generator declaration", interner)?; + cursor.expect( + (Keyword::Async, false), + "async generator declaration", + interner, + )?; cursor.peek_expect_no_lineterminator(0, "async generator declaration", interner)?; - cursor.expect(Keyword::Function, "async generator declaration", interner)?; + cursor.expect( + (Keyword::Function, false), + "async generator declaration", + interner, + )?; cursor.expect(Punctuator::Mul, "async generator declaration", interner)?; let result = parse_callable_declaration(&self, cursor, interner)?; diff --git a/boa_engine/src/syntax/parser/statement/declaration/hoistable/class_decl/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/hoistable/class_decl/mod.rs index 920ed1af0e..4f72be44ad 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/hoistable/class_decl/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/hoistable/class_decl/mod.rs @@ -69,13 +69,13 @@ where cursor: &mut Cursor, interner: &mut Interner, ) -> Result { - cursor.expect(Keyword::Class, "class declaration", interner)?; + cursor.expect((Keyword::Class, false), "class declaration", interner)?; let strict = cursor.strict_mode(); cursor.set_strict_mode(true); let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; let name = match token.kind() { - TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => { + TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => { BindingIdentifier::new(self.allow_yield, self.allow_await) .parse(cursor, interner)? } @@ -136,12 +136,17 @@ where interner: &mut Interner, ) -> Result { let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; - let super_ref = if token.kind() == &TokenKind::Keyword(Keyword::Extends) { - Some(Box::new( + let super_ref = match token.kind() { + TokenKind::Keyword((Keyword::Extends, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Extends, false)) => Some(Box::new( ClassHeritage::new(self.allow_yield, self.allow_await).parse(cursor, interner)?, - )) - } else { - None + )), + _ => None, }; cursor.expect(Punctuator::OpenBlock, "class tail", interner)?; @@ -207,7 +212,7 @@ where interner: &mut Interner, ) -> Result { cursor.expect( - TokenKind::Keyword(Keyword::Extends), + TokenKind::Keyword((Keyword::Extends, false)), "class heritage", interner, )?; @@ -592,7 +597,13 @@ where ClassElementNode::MethodDefinition(property_name, method) } } - TokenKind::Keyword(Keyword::Async) => { + TokenKind::Keyword((Keyword::Async, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Async, false)) => { cursor.next(interner).expect("token disappeared"); cursor.peek_expect_no_lineterminator(0, "Async object methods", interner)?; let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; diff --git a/boa_engine/src/syntax/parser/statement/declaration/hoistable/function_decl/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/hoistable/function_decl/mod.rs index 14d526164e..7a62c28c8f 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/hoistable/function_decl/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/hoistable/function_decl/mod.rs @@ -85,7 +85,7 @@ where cursor: &mut Cursor, interner: &mut Interner, ) -> Result { - cursor.expect(Keyword::Function, "function declaration", interner)?; + cursor.expect((Keyword::Function, false), "function declaration", interner)?; let result = parse_callable_declaration(&self, cursor, interner)?; diff --git a/boa_engine/src/syntax/parser/statement/declaration/hoistable/generator_decl/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/hoistable/generator_decl/mod.rs index 40c4d1166d..7e546b6388 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/hoistable/generator_decl/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/hoistable/generator_decl/mod.rs @@ -80,7 +80,11 @@ where cursor: &mut Cursor, interner: &mut Interner, ) -> Result { - cursor.expect(Keyword::Function, "generator declaration", interner)?; + cursor.expect( + (Keyword::Function, false), + "generator declaration", + interner, + )?; cursor.expect(Punctuator::Mul, "generator declaration", interner)?; let result = parse_callable_declaration(&self, cursor, interner)?; diff --git a/boa_engine/src/syntax/parser/statement/declaration/hoistable/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/hoistable/mod.rs index c948fd63b4..c3eeeedc1c 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/hoistable/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/hoistable/mod.rs @@ -75,7 +75,13 @@ where let tok = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { - TokenKind::Keyword(Keyword::Function) => { + TokenKind::Keyword((Keyword::Function | Keyword::Async | Keyword::Class, true)) => { + Err(ParseError::general( + "Keyword must not contain escaped characters", + tok.span().start(), + )) + } + TokenKind::Keyword((Keyword::Function, false)) => { let next_token = cursor.peek(1, interner)?.ok_or(ParseError::AbruptEnd)?; if let TokenKind::Punctuator(Punctuator::Mul) = next_token.kind() { GeneratorDeclaration::new(self.allow_yield, self.allow_await, self.is_default) @@ -87,7 +93,7 @@ where .map(Node::from) } } - TokenKind::Keyword(Keyword::Async) => { + TokenKind::Keyword((Keyword::Async, false)) => { let next_token = cursor.peek(2, interner)?.ok_or(ParseError::AbruptEnd)?; if let TokenKind::Punctuator(Punctuator::Mul) = next_token.kind() { AsyncGeneratorDeclaration::new( @@ -103,7 +109,7 @@ where .map(Node::from) } } - TokenKind::Keyword(Keyword::Class) => { + TokenKind::Keyword((Keyword::Class, false)) => { ClassDeclaration::new(false, false, self.is_default) .parse(cursor, interner) .map(Node::from) diff --git a/boa_engine/src/syntax/parser/statement/declaration/lexical.rs b/boa_engine/src/syntax/parser/statement/declaration/lexical.rs index f8bc55e55d..f336867972 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/lexical.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/lexical.rs @@ -74,7 +74,11 @@ where let tok = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { - TokenKind::Keyword(Keyword::Const) => BindingList::new( + TokenKind::Keyword((Keyword::Const | Keyword::Let, true)) => Err(ParseError::general( + "Keyword must not contain escaped characters", + tok.span().start(), + )), + TokenKind::Keyword((Keyword::Const, false)) => BindingList::new( self.allow_in, self.allow_yield, self.allow_await, @@ -82,7 +86,7 @@ where self.const_init_required, ) .parse(cursor, interner), - TokenKind::Keyword(Keyword::Let) => BindingList::new( + TokenKind::Keyword((Keyword::Let, false)) => BindingList::new( self.allow_in, self.allow_yield, self.allow_await, @@ -182,8 +186,17 @@ where match cursor.peek_semicolon(interner)? { SemicolonResult::Found(_) => break, SemicolonResult::NotFound(tk) - if tk.kind() == &TokenKind::Keyword(Keyword::Of) - || tk.kind() == &TokenKind::Keyword(Keyword::In) => + if tk.kind() == &TokenKind::Keyword((Keyword::Of, true)) + || tk.kind() == &TokenKind::Keyword((Keyword::In, true)) => + { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + tk.span().start(), + )); + } + SemicolonResult::NotFound(tk) + if tk.kind() == &TokenKind::Keyword((Keyword::Of, false)) + || tk.kind() == &TokenKind::Keyword((Keyword::In, false)) => { break } diff --git a/boa_engine/src/syntax/parser/statement/declaration/mod.rs b/boa_engine/src/syntax/parser/statement/declaration/mod.rs index e42195c2b9..70a7bc8e96 100644 --- a/boa_engine/src/syntax/parser/statement/declaration/mod.rs +++ b/boa_engine/src/syntax/parser/statement/declaration/mod.rs @@ -67,11 +67,11 @@ where let tok = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { - TokenKind::Keyword(Keyword::Function | Keyword::Async | Keyword::Class) => { + TokenKind::Keyword((Keyword::Function | Keyword::Async | Keyword::Class, _)) => { HoistableDeclaration::new(self.allow_yield, self.allow_await, false) .parse(cursor, interner) } - TokenKind::Keyword(Keyword::Const | Keyword::Let) => LexicalDeclaration::new( + TokenKind::Keyword((Keyword::Const | Keyword::Let, _)) => LexicalDeclaration::new( true, self.allow_yield, self.allow_await, diff --git a/boa_engine/src/syntax/parser/statement/expression/mod.rs b/boa_engine/src/syntax/parser/statement/expression/mod.rs index 8d9659a94c..2ce1fa02bf 100644 --- a/boa_engine/src/syntax/parser/statement/expression/mod.rs +++ b/boa_engine/src/syntax/parser/statement/expression/mod.rs @@ -47,22 +47,40 @@ where let next_token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; match next_token.kind() { - TokenKind::Keyword(Keyword::Function | Keyword::Class) => { + TokenKind::Keyword(( + Keyword::Function | Keyword::Class | Keyword::Async | Keyword::Let, + true, + )) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + next_token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Function | Keyword::Class, false)) => { return Err(ParseError::general( "expected statement", next_token.span().start(), )); } - TokenKind::Keyword(Keyword::Async) => { + TokenKind::Keyword((Keyword::Async, false)) => { let next_token = cursor.peek(1, interner)?.ok_or(ParseError::AbruptEnd)?; - if next_token.kind() == &TokenKind::Keyword(Keyword::Function) { - return Err(ParseError::general( - "expected statement", - next_token.span().start(), - )); + match next_token.kind() { + TokenKind::Keyword((Keyword::Function, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + next_token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Function, false)) => { + return Err(ParseError::general( + "expected statement", + next_token.span().start(), + )); + } + _ => {} } } - TokenKind::Keyword(Keyword::Let) => { + TokenKind::Keyword((Keyword::Let, false)) => { let next_token = cursor.peek(1, interner)?.ok_or(ParseError::AbruptEnd)?; if next_token.kind() == &TokenKind::Punctuator(Punctuator::OpenBracket) { return Err(ParseError::general( diff --git a/boa_engine/src/syntax/parser/statement/if_stm/mod.rs b/boa_engine/src/syntax/parser/statement/if_stm/mod.rs index df658cf37b..66e051c0e2 100644 --- a/boa_engine/src/syntax/parser/statement/if_stm/mod.rs +++ b/boa_engine/src/syntax/parser/statement/if_stm/mod.rs @@ -60,7 +60,7 @@ where ) -> Result { let _timer = Profiler::global().start_event("IfStatement", "Parsing"); - cursor.expect(Keyword::If, "if statement", interner)?; + cursor.expect((Keyword::If, false), "if statement", interner)?; cursor.expect(Punctuator::OpenParen, "if statement", interner)?; let condition = Expression::new(None, true, self.allow_yield, self.allow_await) @@ -71,61 +71,72 @@ where .span() .end(); - let then_node = if !cursor.strict_mode() - && cursor - .peek(0, interner)? - .ok_or(ParseError::AbruptEnd)? - .kind() - == &TokenKind::Keyword(Keyword::Function) - { - // FunctionDeclarations in IfStatement Statement Clauses - // https://tc39.es/ecma262/#sec-functiondeclarations-in-ifstatement-statement-clauses - FunctionDeclaration::new(self.allow_yield, self.allow_await, false) - .parse(cursor, interner)? - .into() - } else { - let node = Statement::new(self.allow_yield, self.allow_await, self.allow_return) - .parse(cursor, interner)?; - - // Early Error: It is a Syntax Error if IsLabelledFunction(the first Statement) is true. - if let Node::FunctionDecl(_) = node { - return Err(ParseError::wrong_function_declaration_non_strict(position)); - } - - node - }; - - let else_node = if cursor.next_if(Keyword::Else, interner)?.is_some() { - let position = cursor - .peek(0, interner)? - .ok_or(ParseError::AbruptEnd)? - .span() - .start(); - - if !cursor.strict_mode() - && cursor - .peek(0, interner)? - .ok_or(ParseError::AbruptEnd)? - .kind() - == &TokenKind::Keyword(Keyword::Function) - { + let strict = cursor.strict_mode(); + let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; + let then_node = match token.kind() { + TokenKind::Keyword((Keyword::Function, _)) if !strict => { // FunctionDeclarations in IfStatement Statement Clauses // https://tc39.es/ecma262/#sec-functiondeclarations-in-ifstatement-statement-clauses - Some( - FunctionDeclaration::new(self.allow_yield, self.allow_await, false) - .parse(cursor, interner)? - .into(), - ) - } else { + FunctionDeclaration::new(self.allow_yield, self.allow_await, false) + .parse(cursor, interner)? + .into() + } + _ => { let node = Statement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner)?; - // Early Error: It is a Syntax Error if IsLabelledFunction(the second Statement) is true. + // Early Error: It is a Syntax Error if IsLabelledFunction(the first Statement) is true. if let Node::FunctionDecl(_) = node { return Err(ParseError::wrong_function_declaration_non_strict(position)); } - Some(node) + node + } + }; + + let else_node = if let Some(token) = cursor.peek(0, interner)? { + match token.kind() { + TokenKind::Keyword((Keyword::Else, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Else, false)) => { + cursor.next(interner)?.expect("token disappeared"); + + let strict = cursor.strict_mode(); + let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; + match token.kind() { + TokenKind::Keyword((Keyword::Function, _)) if !strict => { + // FunctionDeclarations in IfStatement Statement Clauses + // https://tc39.es/ecma262/#sec-functiondeclarations-in-ifstatement-statement-clauses + Some( + FunctionDeclaration::new(self.allow_yield, self.allow_await, false) + .parse(cursor, interner)? + .into(), + ) + } + _ => { + let node = Statement::new( + self.allow_yield, + self.allow_await, + self.allow_return, + ) + .parse(cursor, interner)?; + + // Early Error: It is a Syntax Error if IsLabelledFunction(the second Statement) is true. + if let Node::FunctionDecl(_) = node { + return Err(ParseError::wrong_function_declaration_non_strict( + position, + )); + } + + Some(node) + } + } + } + _ => None, } } else { None diff --git a/boa_engine/src/syntax/parser/statement/iteration/do_while_statement.rs b/boa_engine/src/syntax/parser/statement/iteration/do_while_statement.rs index b8cd820114..81afce10fd 100644 --- a/boa_engine/src/syntax/parser/statement/iteration/do_while_statement.rs +++ b/boa_engine/src/syntax/parser/statement/iteration/do_while_statement.rs @@ -68,7 +68,7 @@ where let _timer = Profiler::global().start_event("DoWhileStatement", "Parsing"); let position = cursor - .expect(Keyword::Do, "do while statement", interner)? + .expect((Keyword::Do, false), "do while statement", interner)? .span() .end(); @@ -81,17 +81,25 @@ where } let next_token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; - - if next_token.kind() != &TokenKind::Keyword(Keyword::While) { - return Err(ParseError::expected( - ["while".to_owned()], - next_token.to_string(interner), - next_token.span(), - "do while statement", - )); + match next_token.kind() { + TokenKind::Keyword((Keyword::While, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + next_token.span().start(), + )); + } + TokenKind::Keyword((Keyword::While, false)) => {} + _ => { + return Err(ParseError::expected( + ["while".to_owned()], + next_token.to_string(interner), + next_token.span(), + "do while statement", + )); + } } - cursor.expect(Keyword::While, "do while statement", interner)?; + cursor.expect((Keyword::While, false), "do while statement", interner)?; cursor.expect(Punctuator::OpenParen, "do while statement", interner)?; diff --git a/boa_engine/src/syntax/parser/statement/iteration/for_statement.rs b/boa_engine/src/syntax/parser/statement/iteration/for_statement.rs index 55743388b7..47f4f5166a 100644 --- a/boa_engine/src/syntax/parser/statement/iteration/for_statement.rs +++ b/boa_engine/src/syntax/parser/statement/iteration/for_statement.rs @@ -77,7 +77,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("ForStatement", "Parsing"); - cursor.expect(Keyword::For, "for statement", interner)?; + cursor.expect((Keyword::For, false), "for statement", interner)?; let init_position = cursor .expect(Punctuator::OpenParen, "for statement", interner)? .span() @@ -88,7 +88,7 @@ where .ok_or(ParseError::AbruptEnd)? .kind() { - TokenKind::Keyword(Keyword::Var) => { + TokenKind::Keyword((Keyword::Var, _)) => { let _next = cursor.next(interner)?; Some( VariableDeclarationList::new(false, self.allow_yield, self.allow_await) @@ -96,7 +96,7 @@ where .map(Node::from)?, ) } - TokenKind::Keyword(Keyword::Let | Keyword::Const) => Some( + TokenKind::Keyword((Keyword::Let | Keyword::Const, _)) => Some( Declaration::new(self.allow_yield, self.allow_await, false) .parse(cursor, interner)?, ), @@ -107,8 +107,15 @@ where ), }; - match (init.as_ref(), cursor.peek(0, interner)?) { - (Some(init), Some(tok)) if tok.kind() == &TokenKind::Keyword(Keyword::In) => { + let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; + match (init.as_ref(), token.kind()) { + (Some(_), TokenKind::Keyword((Keyword::In | Keyword::Of, true))) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + (Some(init), TokenKind::Keyword((Keyword::In, false))) => { let init = node_to_iterable_loop_initializer(init, init_position)?; let _next = cursor.next(interner)?; @@ -130,7 +137,7 @@ where return Ok(ForInLoop::new(init, expr, body).into()); } - (Some(init), Some(tok)) if tok.kind() == &TokenKind::Keyword(Keyword::Of) => { + (Some(init), TokenKind::Keyword((Keyword::Of, false))) => { let init = node_to_iterable_loop_initializer(init, init_position)?; let _next = cursor.next(interner)?; diff --git a/boa_engine/src/syntax/parser/statement/iteration/while_statement.rs b/boa_engine/src/syntax/parser/statement/iteration/while_statement.rs index 479030fa10..0fb46dabfb 100644 --- a/boa_engine/src/syntax/parser/statement/iteration/while_statement.rs +++ b/boa_engine/src/syntax/parser/statement/iteration/while_statement.rs @@ -56,7 +56,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("WhileStatement", "Parsing"); - cursor.expect(Keyword::While, "while statement", interner)?; + cursor.expect((Keyword::While, false), "while statement", interner)?; cursor.expect(Punctuator::OpenParen, "while statement", interner)?; diff --git a/boa_engine/src/syntax/parser/statement/labelled_stm/mod.rs b/boa_engine/src/syntax/parser/statement/labelled_stm/mod.rs index 9665388761..36c6d87dca 100644 --- a/boa_engine/src/syntax/parser/statement/labelled_stm/mod.rs +++ b/boa_engine/src/syntax/parser/statement/labelled_stm/mod.rs @@ -69,13 +69,13 @@ where // Early Error: It is a Syntax Error if any strict mode source code matches this rule. // https://tc39.es/ecma262/#sec-labelled-statements-static-semantics-early-errors // https://tc39.es/ecma262/#sec-labelled-function-declarations - TokenKind::Keyword(Keyword::Function) if strict => { + TokenKind::Keyword((Keyword::Function, _)) if strict => { return Err(ParseError::general( "In strict mode code, functions can only be declared at top level or inside a block.", next_token.span().start() )) } - TokenKind::Keyword(Keyword::Function) => { + TokenKind::Keyword((Keyword::Function, _)) => { FunctionDeclaration::new(self.allow_yield, self.allow_await, false) .parse(cursor, interner)? .into() diff --git a/boa_engine/src/syntax/parser/statement/mod.rs b/boa_engine/src/syntax/parser/statement/mod.rs index 242263639d..bda1529a47 100644 --- a/boa_engine/src/syntax/parser/statement/mod.rs +++ b/boa_engine/src/syntax/parser/statement/mod.rs @@ -126,35 +126,35 @@ where let tok = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; match tok.kind() { - TokenKind::Keyword(Keyword::Await) => AwaitExpression::new(self.allow_yield) + TokenKind::Keyword((Keyword::Await, _)) => AwaitExpression::new(self.allow_yield) .parse(cursor, interner) .map(Node::from), - TokenKind::Keyword(Keyword::If) => { + TokenKind::Keyword((Keyword::If, _)) => { IfStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Var) => { + TokenKind::Keyword((Keyword::Var, _)) => { VariableStatement::new(self.allow_yield, self.allow_await) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::While) => { + TokenKind::Keyword((Keyword::While, _)) => { WhileStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Do) => { + TokenKind::Keyword((Keyword::Do, _)) => { DoWhileStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::For) => { + TokenKind::Keyword((Keyword::For, _)) => { ForStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Return) => { + TokenKind::Keyword((Keyword::Return, _)) => { if self.allow_return.0 { ReturnStatement::new(self.allow_yield, self.allow_await) .parse(cursor, interner) @@ -167,27 +167,27 @@ where )) } } - TokenKind::Keyword(Keyword::Break) => { + TokenKind::Keyword((Keyword::Break, _)) => { BreakStatement::new(self.allow_yield, self.allow_await) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Continue) => { + TokenKind::Keyword((Keyword::Continue, _)) => { ContinueStatement::new(self.allow_yield, self.allow_await) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Try) => { + TokenKind::Keyword((Keyword::Try, _)) => { TryStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Throw) => { + TokenKind::Keyword((Keyword::Throw, _)) => { ThrowStatement::new(self.allow_yield, self.allow_await) .parse(cursor, interner) .map(Node::from) } - TokenKind::Keyword(Keyword::Switch) => { + TokenKind::Keyword((Keyword::Switch, _)) => { SwitchStatement::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner) .map(Node::from) @@ -471,7 +471,7 @@ where let tok = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; match *tok.kind() { - TokenKind::Keyword(Keyword::Function | Keyword::Async | Keyword::Class) => { + TokenKind::Keyword((Keyword::Function | Keyword::Async | Keyword::Class, _)) => { if strict_mode && self.in_block { return Err(ParseError::lex(LexError::Syntax( "Function declaration in blocks not allowed in strict mode".into(), @@ -480,7 +480,7 @@ where } Declaration::new(self.allow_yield, self.allow_await, true).parse(cursor, interner) } - TokenKind::Keyword(Keyword::Const | Keyword::Let) => { + TokenKind::Keyword((Keyword::Const | Keyword::Let, _)) => { Declaration::new(self.allow_yield, self.allow_await, true).parse(cursor, interner) } _ => Statement::new(self.allow_yield, self.allow_await, self.allow_return) @@ -555,14 +555,14 @@ where ))) } TokenKind::Identifier(ref s) => Ok(*s), - TokenKind::Keyword(Keyword::Yield) if self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => { // Early Error: It is a Syntax Error if this production has a [Yield] parameter and StringValue of Identifier is "yield". Err(ParseError::general( "Unexpected identifier", next_token.span().start(), )) } - TokenKind::Keyword(Keyword::Yield) if !self.allow_yield.0 => { + TokenKind::Keyword((Keyword::Yield, _)) if !self.allow_yield.0 => { if cursor.strict_mode() { Err(ParseError::general( "yield keyword in binding identifier not allowed in strict mode", @@ -572,15 +572,15 @@ where Ok(Sym::YIELD) } } - TokenKind::Keyword(Keyword::Await) if cursor.arrow() => Ok(Sym::AWAIT), - TokenKind::Keyword(Keyword::Await) if self.allow_await.0 => { + TokenKind::Keyword((Keyword::Await, _)) if cursor.arrow() => Ok(Sym::AWAIT), + TokenKind::Keyword((Keyword::Await, _)) if self.allow_await.0 => { // Early Error: It is a Syntax Error if this production has an [Await] parameter and StringValue of Identifier is "await". Err(ParseError::general( "Unexpected identifier", next_token.span().start(), )) } - TokenKind::Keyword(Keyword::Await) if !self.allow_await.0 => { + TokenKind::Keyword((Keyword::Await, _)) if !self.allow_await.0 => { if cursor.strict_mode() { Err(ParseError::general( "await keyword in binding identifier not allowed in strict mode", diff --git a/boa_engine/src/syntax/parser/statement/return_stm/mod.rs b/boa_engine/src/syntax/parser/statement/return_stm/mod.rs index eb761f2abc..5cd3f1a46f 100644 --- a/boa_engine/src/syntax/parser/statement/return_stm/mod.rs +++ b/boa_engine/src/syntax/parser/statement/return_stm/mod.rs @@ -51,7 +51,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("ReturnStatement", "Parsing"); - cursor.expect(Keyword::Return, "return statement", interner)?; + cursor.expect((Keyword::Return, false), "return statement", interner)?; if let SemicolonResult::Found(tok) = cursor.peek_semicolon(interner)? { match tok { diff --git a/boa_engine/src/syntax/parser/statement/switch/mod.rs b/boa_engine/src/syntax/parser/statement/switch/mod.rs index 709f9f804e..7fe3324418 100644 --- a/boa_engine/src/syntax/parser/statement/switch/mod.rs +++ b/boa_engine/src/syntax/parser/statement/switch/mod.rs @@ -16,8 +16,8 @@ use std::io::Read; /// The possible `TokenKind` which indicate the end of a case statement. const CASE_BREAK_TOKENS: [TokenKind; 3] = [ TokenKind::Punctuator(Punctuator::CloseBlock), - TokenKind::Keyword(Keyword::Case), - TokenKind::Keyword(Keyword::Default), + TokenKind::Keyword((Keyword::Case, false)), + TokenKind::Keyword((Keyword::Default, false)), ]; /// Switch statement parsing. @@ -63,7 +63,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("SwitchStatement", "Parsing"); - cursor.expect(Keyword::Switch, "switch statement", interner)?; + cursor.expect((Keyword::Switch, false), "switch statement", interner)?; cursor.expect(Punctuator::OpenParen, "switch statement", interner)?; let condition = Expression::new(None, true, self.allow_yield, self.allow_await) @@ -125,8 +125,15 @@ where let mut default = None; loop { - match cursor.next(interner)? { - Some(token) if token.kind() == &TokenKind::Keyword(Keyword::Case) => { + let token = cursor.next(interner)?.ok_or(ParseError::AbruptEnd)?; + match token.kind() { + TokenKind::Keyword((Keyword::Case | Keyword::Default, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Case, false)) => { // Case statement. let cond = Expression::new(None, true, self.allow_yield, self.allow_await) .parse(cursor, interner)?; @@ -144,7 +151,7 @@ where cases.push(node::Case::new(cond, statement_list)); } - Some(token) if token.kind() == &TokenKind::Keyword(Keyword::Default) => { + TokenKind::Keyword((Keyword::Default, false)) => { if default.is_some() { // If default has already been defined then it cannot be defined again and to do so is an error. return Err(ParseError::unexpected( @@ -167,10 +174,8 @@ where default = Some(statement_list); } - Some(token) if token.kind() == &TokenKind::Punctuator(Punctuator::CloseBlock) => { - break - } - Some(token) => { + TokenKind::Punctuator(Punctuator::CloseBlock) => break, + _ => { return Err(ParseError::expected( ["case".to_owned(), "default".to_owned(), "}".to_owned()], token.to_string(interner), @@ -178,7 +183,6 @@ where "switch case block", )) } - None => return Err(ParseError::AbruptEnd), } } diff --git a/boa_engine/src/syntax/parser/statement/throw/mod.rs b/boa_engine/src/syntax/parser/statement/throw/mod.rs index 67b181a8c1..f7cbc3468c 100644 --- a/boa_engine/src/syntax/parser/statement/throw/mod.rs +++ b/boa_engine/src/syntax/parser/statement/throw/mod.rs @@ -50,7 +50,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("ThrowStatement", "Parsing"); - cursor.expect(Keyword::Throw, "throw statement", interner)?; + cursor.expect((Keyword::Throw, false), "throw statement", interner)?; cursor.peek_expect_no_lineterminator(0, "throw statement", interner)?; diff --git a/boa_engine/src/syntax/parser/statement/try_stm/catch.rs b/boa_engine/src/syntax/parser/statement/try_stm/catch.rs index 0fd4a2600d..625b5d9b43 100644 --- a/boa_engine/src/syntax/parser/statement/try_stm/catch.rs +++ b/boa_engine/src/syntax/parser/statement/try_stm/catch.rs @@ -57,7 +57,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("Catch", "Parsing"); - cursor.expect(Keyword::Catch, "try statement", interner)?; + cursor.expect((Keyword::Catch, false), "try statement", interner)?; let catch_param = if cursor.next_if(Punctuator::OpenParen, interner)?.is_some() { let catch_param = CatchParameter::new(self.allow_yield, self.allow_await).parse(cursor, interner)?; diff --git a/boa_engine/src/syntax/parser/statement/try_stm/finally.rs b/boa_engine/src/syntax/parser/statement/try_stm/finally.rs index d98ce58fe1..ca5e438753 100644 --- a/boa_engine/src/syntax/parser/statement/try_stm/finally.rs +++ b/boa_engine/src/syntax/parser/statement/try_stm/finally.rs @@ -52,7 +52,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("Finally", "Parsing"); - cursor.expect(Keyword::Finally, "try statement", interner)?; + cursor.expect((Keyword::Finally, false), "try statement", interner)?; Ok( Block::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner)? diff --git a/boa_engine/src/syntax/parser/statement/try_stm/mod.rs b/boa_engine/src/syntax/parser/statement/try_stm/mod.rs index f9eceec294..73cba95889 100644 --- a/boa_engine/src/syntax/parser/statement/try_stm/mod.rs +++ b/boa_engine/src/syntax/parser/statement/try_stm/mod.rs @@ -59,25 +59,31 @@ where ) -> Result { let _timer = Profiler::global().start_event("TryStatement", "Parsing"); // TRY - cursor.expect(Keyword::Try, "try statement", interner)?; + cursor.expect((Keyword::Try, false), "try statement", interner)?; let try_clause = Block::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner)?; let next_token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?; - - if next_token.kind() != &TokenKind::Keyword(Keyword::Catch) - && next_token.kind() != &TokenKind::Keyword(Keyword::Finally) - { - return Err(ParseError::expected( - ["catch".to_owned(), "finally".to_owned()], - next_token.to_string(interner), - next_token.span(), - "try statement", - )); + match next_token.kind() { + TokenKind::Keyword((Keyword::Catch | Keyword::Finally, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + next_token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Catch | Keyword::Finally, false)) => {} + _ => { + return Err(ParseError::expected( + ["catch".to_owned(), "finally".to_owned()], + next_token.to_string(interner), + next_token.span(), + "try statement", + )); + } } - let catch = if next_token.kind() == &TokenKind::Keyword(Keyword::Catch) { + let catch = if next_token.kind() == &TokenKind::Keyword((Keyword::Catch, false)) { Some( Catch::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner)?, @@ -89,7 +95,13 @@ where let next_token = cursor.peek(0, interner)?; let finally_block = if let Some(token) = next_token { match token.kind() { - TokenKind::Keyword(Keyword::Finally) => Some( + TokenKind::Keyword((Keyword::Finally, true)) => { + return Err(ParseError::general( + "Keyword must not contain escaped characters", + token.span().start(), + )); + } + TokenKind::Keyword((Keyword::Finally, false)) => Some( Finally::new(self.allow_yield, self.allow_await, self.allow_return) .parse(cursor, interner)?, ), diff --git a/boa_engine/src/syntax/parser/statement/variable/mod.rs b/boa_engine/src/syntax/parser/statement/variable/mod.rs index f8d932b496..a0179e9a41 100644 --- a/boa_engine/src/syntax/parser/statement/variable/mod.rs +++ b/boa_engine/src/syntax/parser/statement/variable/mod.rs @@ -60,7 +60,7 @@ where interner: &mut Interner, ) -> Result { let _timer = Profiler::global().start_event("VariableStatement", "Parsing"); - cursor.expect(Keyword::Var, "variable statement", interner)?; + cursor.expect((Keyword::Var, false), "variable statement", interner)?; let decl_list = VariableDeclarationList::new(true, self.allow_yield, self.allow_await) .parse(cursor, interner)?;