@ -2,8 +2,9 @@
#![ allow(clippy::indexing_slicing) ]
#![ allow(clippy::indexing_slicing) ]
use crate ::lexer ::{
use crate ::lexer ::{
template ::TemplateString , token ::Numeric , Cursor , Error , Interner , Lexer , Position , Punctuator ,
template ::TemplateString ,
Read , Span , TokenKind ,
token ::{ ContainsEscapeSequence , EscapeSequence , Numeric } ,
Cursor , Error , Interner , Lexer , Position , Punctuator , Read , Span , TokenKind ,
} ;
} ;
use boa_ast ::Keyword ;
use boa_ast ::Keyword ;
use boa_interner ::Sym ;
use boa_interner ::Sym ;
@ -94,9 +95,18 @@ fn check_identifier() {
TokenKind ::identifier (
TokenKind ::identifier (
interner . get_or_intern_static ( "x\u{200C}\u{200D}" , utf16 ! ( "x\u{200C}\u{200D}" ) ) ,
interner . get_or_intern_static ( "x\u{200C}\u{200D}" , utf16 ! ( "x\u{200C}\u{200D}" ) ) ,
) ,
) ,
TokenKind ::identifier ( interner . get_or_intern_static ( "x" , utf16 ! ( "x" ) ) ) ,
TokenKind ::Identifier ( (
TokenKind ::identifier ( interner . get_or_intern_static ( "xx" , utf16 ! ( "xx" ) ) ) ,
interner . get_or_intern_static ( "x" , utf16 ! ( "x" ) ) ,
TokenKind ::identifier ( interner . get_or_intern_static ( "xxx" , utf16 ! ( "xxx" ) ) ) ,
ContainsEscapeSequence ( true ) ,
) ) ,
TokenKind ::Identifier ( (
interner . get_or_intern_static ( "xx" , utf16 ! ( "xx" ) ) ,
ContainsEscapeSequence ( true ) ,
) ) ,
TokenKind ::Identifier ( (
interner . get_or_intern_static ( "xxx" , utf16 ! ( "xxx" ) ) ,
ContainsEscapeSequence ( true ) ,
) ) ,
] ;
] ;
expect_tokens ( & mut lexer , & expected , interner ) ;
expect_tokens ( & mut lexer , & expected , interner ) ;
@ -141,8 +151,8 @@ fn check_string() {
let a_sym = interner . get_or_intern_static ( "aaa" , utf16 ! ( "aaa" ) ) ;
let a_sym = interner . get_or_intern_static ( "aaa" , utf16 ! ( "aaa" ) ) ;
let b_sym = interner . get_or_intern_static ( "bbb" , utf16 ! ( "bbb" ) ) ;
let b_sym = interner . get_or_intern_static ( "bbb" , utf16 ! ( "bbb" ) ) ;
let expected = [
let expected = [
TokenKind ::string_literal ( a_sym ) ,
TokenKind ::string_literal ( a_sym , None ) ,
TokenKind ::string_literal ( b_sym ) ,
TokenKind ::string_literal ( b_sym , None ) ,
] ;
] ;
expect_tokens ( & mut lexer , & expected , interner ) ;
expect_tokens ( & mut lexer , & expected , interner ) ;
@ -305,7 +315,7 @@ fn check_variable_definition_tokens() {
TokenKind ::Keyword ( ( Keyword ::Let , false ) ) ,
TokenKind ::Keyword ( ( Keyword ::Let , false ) ) ,
TokenKind ::identifier ( a_sym ) ,
TokenKind ::identifier ( a_sym ) ,
TokenKind ::Punctuator ( Punctuator ::Assign ) ,
TokenKind ::Punctuator ( Punctuator ::Assign ) ,
TokenKind ::string_literal ( hello_sym ) ,
TokenKind ::string_literal ( hello_sym , None ) ,
TokenKind ::Punctuator ( Punctuator ::Semicolon ) ,
TokenKind ::Punctuator ( Punctuator ::Semicolon ) ,
] ;
] ;
@ -943,7 +953,7 @@ fn string_unicode() {
let sym = interner . get_or_intern_static ( "中文" , utf16 ! ( "中文" ) ) ;
let sym = interner . get_or_intern_static ( "中文" , utf16 ! ( "中文" ) ) ;
let expected = [
let expected = [
TokenKind ::StringLiteral ( sym ) ,
TokenKind ::StringLiteral ( ( sym , None ) ) ,
TokenKind ::Punctuator ( Punctuator ::Semicolon ) ,
TokenKind ::Punctuator ( Punctuator ::Semicolon ) ,
] ;
] ;
@ -957,7 +967,7 @@ fn string_unicode_escape_with_braces() {
let sym =
let sym =
interner . get_or_intern_static ( "{\u{20ac}\u{a0}\u{a0}}" , utf16 ! ( "{\u{20ac}\u{a0}\u{a0}}" ) ) ;
interner . get_or_intern_static ( "{\u{20ac}\u{a0}\u{a0}}" , utf16 ! ( "{\u{20ac}\u{a0}\u{a0}}" ) ) ;
let expected = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected = [ TokenKind ::StringLiteral ( ( sym , None ) ) ] ;
expect_tokens ( & mut lexer , & expected , interner ) ;
expect_tokens ( & mut lexer , & expected , interner ) ;
@ -992,7 +1002,7 @@ fn string_unicode_escape_with_braces_2() {
let interner = & mut Interner ::default ( ) ;
let interner = & mut Interner ::default ( ) ;
let sym = interner . get_or_intern_static ( "\u{20ac}\u{a0}\u{a0}" , utf16 ! ( "\u{20ac}\u{a0}\u{a0}" ) ) ;
let sym = interner . get_or_intern_static ( "\u{20ac}\u{a0}\u{a0}" , utf16 ! ( "\u{20ac}\u{a0}\u{a0}" ) ) ;
let expected = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected = [ TokenKind ::StringLiteral ( ( sym , None ) ) ] ;
expect_tokens ( & mut lexer , & expected , interner ) ;
expect_tokens ( & mut lexer , & expected , interner ) ;
}
}
@ -1005,7 +1015,7 @@ fn string_with_single_escape() {
let interner = & mut Interner ::default ( ) ;
let interner = & mut Interner ::default ( ) ;
let sym = interner . get_or_intern_static ( "Б" , utf16 ! ( "Б" ) ) ;
let sym = interner . get_or_intern_static ( "Б" , utf16 ! ( "Б" ) ) ;
let expected = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected = [ TokenKind ::StringLiteral ( ( sym , None ) ) ] ;
expect_tokens ( & mut lexer , & expected , interner ) ;
expect_tokens ( & mut lexer , & expected , interner ) ;
}
}
@ -1027,7 +1037,10 @@ fn string_legacy_octal_escape() {
let interner = & mut Interner ::default ( ) ;
let interner = & mut Interner ::default ( ) ;
let sym = interner . get_or_intern ( expected . encode_utf16 ( ) . collect ::< Vec < _ > > ( ) . as_slice ( ) ) ;
let sym = interner . get_or_intern ( expected . encode_utf16 ( ) . collect ::< Vec < _ > > ( ) . as_slice ( ) ) ;
let expected_tokens = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected_tokens = [ TokenKind ::StringLiteral ( (
sym ,
Some ( EscapeSequence ::LegacyOctal ) ,
) ) ] ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
}
}
@ -1057,7 +1070,7 @@ fn string_zero_escape() {
let interner = & mut Interner ::default ( ) ;
let interner = & mut Interner ::default ( ) ;
let sym = interner . get_or_intern ( expected . encode_utf16 ( ) . collect ::< Vec < _ > > ( ) . as_slice ( ) ) ;
let sym = interner . get_or_intern ( expected . encode_utf16 ( ) . collect ::< Vec < _ > > ( ) . as_slice ( ) ) ;
let expected_tokens = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected_tokens = [ TokenKind ::StringLiteral ( ( sym , None ) ) ] ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
}
}
@ -1072,7 +1085,10 @@ fn string_non_octal_decimal_escape() {
let interner = & mut Interner ::default ( ) ;
let interner = & mut Interner ::default ( ) ;
let sym = interner . get_or_intern ( expected . encode_utf16 ( ) . collect ::< Vec < _ > > ( ) . as_slice ( ) ) ;
let sym = interner . get_or_intern ( expected . encode_utf16 ( ) . collect ::< Vec < _ > > ( ) . as_slice ( ) ) ;
let expected_tokens = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected_tokens = [ TokenKind ::StringLiteral ( (
sym ,
Some ( EscapeSequence ::NonOctalDecimal ) ,
) ) ] ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
}
}
@ -1101,7 +1117,7 @@ fn string_line_continuation() {
let interner = & mut Interner ::default ( ) ;
let interner = & mut Interner ::default ( ) ;
let sym = interner . get_or_intern_static ( "hello world" , utf16 ! ( "hello world" ) ) ;
let sym = interner . get_or_intern_static ( "hello world" , utf16 ! ( "hello world" ) ) ;
let expected_tokens = [ TokenKind ::StringLiteral ( sym ) ] ;
let expected_tokens = [ TokenKind ::StringLiteral ( ( sym , None ) ) ] ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
expect_tokens ( & mut lexer , & expected_tokens , interner ) ;
}
}