Browse Source

Fix lexer span panic with carriage return (#799)

Switch to treating bare carriage return as an OS9-style
newline, like v8.
pull/806/head
Theia Vogel 4 years ago committed by GitHub
parent
commit
7a52e269e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 16
      boa/src/syntax/lexer/cursor.rs
  2. 45
      boa/src/syntax/lexer/tests.rs

16
boa/src/syntax/lexer/cursor.rs

@ -33,13 +33,6 @@ impl<R> Cursor<R> {
self.pos = Position::new(next_line, 1);
}
/// Performs a carriage return to modify the position in the source.
#[inline]
fn carriage_return(&mut self) {
let current_line = self.pos.line_number();
self.pos = Position::new(current_line, 1);
}
#[inline]
pub(super) fn strict_mode(&self) -> bool {
self.strict_mode
@ -177,7 +170,14 @@ where
};
match chr {
Some('\r') => self.carriage_return(),
Some('\r') => {
// Try to take a newline if it's next, for windows "\r\n" newlines
// Otherwise, treat as a Mac OS9 bare '\r' newline
if self.peek()? == Some('\n') {
self.peeked.take();
}
self.next_line();
}
Some('\n') | Some('\u{2028}') | Some('\u{2029}') => self.next_line(),
Some(_) => self.next_column(),
None => {}

45
boa/src/syntax/lexer/tests.rs

@ -657,3 +657,48 @@ fn non_english_str() {
expect_tokens(&mut lexer, &expected);
}
mod carriage_return {
use super::*;
fn expect_tokens_with_lines(lines: usize, src: &str) {
let mut lexer = Lexer::new(src.as_bytes());
let mut expected = Vec::with_capacity(lines + 2);
expected.push(TokenKind::Punctuator(Punctuator::Sub));
for _ in 0..lines {
expected.push(TokenKind::LineTerminator);
}
expected.push(TokenKind::NumericLiteral(Numeric::Integer(3)));
expect_tokens(&mut lexer, &expected);
}
#[test]
fn regular_line() {
expect_tokens_with_lines(1, "-\n3");
expect_tokens_with_lines(2, "-\n\n3");
expect_tokens_with_lines(3, "-\n\n\n3");
}
#[test]
fn carriage_return() {
expect_tokens_with_lines(1, "-\r3");
expect_tokens_with_lines(2, "-\r\r3");
expect_tokens_with_lines(3, "-\r\r\r3");
}
#[test]
fn windows_line() {
expect_tokens_with_lines(1, "-\r\n3");
expect_tokens_with_lines(2, "-\r\n\r\n3");
expect_tokens_with_lines(3, "-\r\n\r\n\r\n3");
}
#[test]
fn mixed_line() {
expect_tokens_with_lines(2, "-\r\n\n3");
expect_tokens_with_lines(2, "-\n\r3");
expect_tokens_with_lines(3, "-\r\n\n\r3");
}
}

Loading…
Cancel
Save