Browse Source

Fix single line comment lexing with CRLF line ending (#964)

Co-authored-by: tofpie <tofpie@users.noreply.github.com>
pull/966/head
tofpie 4 years ago committed by GitHub
parent
commit
8f590d781a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      boa/src/syntax/lexer/comment.rs
  2. 15
      boa/src/syntax/lexer/tests.rs

4
boa/src/syntax/lexer/comment.rs

@ -31,11 +31,11 @@ impl<R> Tokenizer<R> for SingleLineComment {
// Skip either to the end of the line or to the end of the input // Skip either to the end of the line or to the end of the input
while let Some(ch) = cursor.peek()? { while let Some(ch) = cursor.peek()? {
if ch == b'\n' { if ch == b'\n' || ch == b'\r' {
break; break;
} else { } else {
// Consume char. // Consume char.
cursor.next_byte()?.expect("Comment character vansihed"); cursor.next_byte()?.expect("Comment character vanished");
} }
} }
Ok(Token::new( Ok(Token::new(

15
boa/src/syntax/lexer/tests.rs

@ -41,6 +41,21 @@ fn check_single_line_comment() {
expect_tokens(&mut lexer, &expected); expect_tokens(&mut lexer, &expected);
} }
#[test]
fn check_single_line_comment_with_crlf_ending() {
let s1 = "var \r\n//This is a comment\r\ntrue";
let mut lexer = Lexer::new(s1.as_bytes());
let expected = [
TokenKind::Keyword(Keyword::Var),
TokenKind::LineTerminator,
TokenKind::LineTerminator,
TokenKind::BooleanLiteral(true),
];
expect_tokens(&mut lexer, &expected);
}
#[test] #[test]
fn check_multi_line_comment() { fn check_multi_line_comment() {
let s = "var /* await \n break \n*/ x"; let s = "var /* await \n break \n*/ x";

Loading…
Cancel
Save