diff --git a/boa/src/lib.rs b/boa/src/lib.rs index aa31a174e3..4eb51f24b8 100644 --- a/boa/src/lib.rs +++ b/boa/src/lib.rs @@ -50,7 +50,6 @@ fn parser_expr(src: &str) -> Result { let mut lexer = Lexer::new(src); lexer.lex().map_err(|e| format!("SyntaxError: {}", e))?; let tokens = lexer.tokens; - // dbg!(&tokens); Parser::new(&tokens) .parse_all() .map_err(|e| format!("ParsingError: {}", e)) diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index faeb02930a..efb81c9160 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -23,6 +23,7 @@ macro_rules! vop { match preview { '=' => { $this.next(); + $this.column_number += 1; $assign_op } _ => $op, @@ -33,10 +34,12 @@ macro_rules! vop { match preview { '=' => { $this.next(); + $this.column_number += 1; $assign_op }, $($case => { $this.next(); + $this.column_number += 1; $block })+, _ => $op @@ -47,6 +50,7 @@ macro_rules! vop { match preview { $($case => { $this.next()?; + $this.column_number += 1; $block })+, _ => $op @@ -135,6 +139,7 @@ impl<'a> Lexer<'a> { buffer: buffer.chars().peekable(), } } + /// Push tokens onto the token queue fn push_token(&mut self, tk: TokenKind) { self.tokens @@ -148,16 +153,17 @@ impl<'a> Lexer<'a> { /// next fetches the next token and return it, or a LexerError if there are no more. fn next(&mut self) -> char { - match self.buffer.next() { - Some(ch) => ch, - None => panic!("No more more characters to consume from input stream, use preview_next() first to check before calling next()"), - } + self.buffer.next().expect( + "No more more characters to consume from input stream, \ + use preview_next() first to check before calling next()", + ) } /// Preview the next character but don't actually increment fn preview_next(&mut self) -> Option { self.buffer.peek().copied() } + /// Preview a char x indexes further in buf, without incrementing fn preview_multiple_next(&mut self, nb_next: usize) -> Option { let mut next_peek = None; @@ -189,7 +195,7 @@ impl<'a> Lexer<'a> { Ok(s) } - /// next_is compares the character passed in to the next character, if they match true is returned and the buffer is incremented + /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented fn next_is(&mut self, peek: char) -> bool { let result = self.preview_next() == Some(peek); if result { @@ -563,6 +569,7 @@ impl<'a> Lexer<'a> { let mut body = String::new(); let mut regex = false; loop { + self.column_number +=1; match self.buffer.next() { // end of body Some('/') => { @@ -570,8 +577,18 @@ impl<'a> Lexer<'a> { break; } // newline/eof not allowed in regex literal - Some('\n') | Some('\r') | Some('\u{2028}') - | Some('\u{2029}') | None => break, + n @ Some('\n') | n @ Some('\r') | n @ Some('\u{2028}') + | n @ Some('\u{2029}') => { + self.column_number = 0; + if n != Some('\r') { + self.line_number += 1; + } + break + }, + None => { + self.column_number -= 1; + break + } // escape sequence Some('\\') => { body.push('\\'); diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index e21e443ae4..98859fe43c 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -330,6 +330,18 @@ fn check_positions() { assert_eq!(lexer.tokens[6].pos.line_number, 1); } +#[test] +#[ignore] +fn test_two_divisions_in_expression() { + let s = " return a !== 0 || 1 / a === 1 / b;"; + let mut lexer = Lexer::new(s); + lexer.lex().expect("failed to lex"); + dbg!(&lexer.tokens); + + assert_eq!(lexer.tokens[11].pos.column_number, 37); + assert_eq!(lexer.tokens[11].pos.line_number, 1); +} + #[test] fn check_line_numbers() { let s = "x\ny\n"; diff --git a/boa/src/syntax/parser/mod.rs b/boa/src/syntax/parser/mod.rs index f06bcb4174..f9138780a3 100644 --- a/boa/src/syntax/parser/mod.rs +++ b/boa/src/syntax/parser/mod.rs @@ -222,28 +222,6 @@ impl<'a> Parser<'a> { } } - /// Returns an error if the next symbol is not `tk` - fn expect_no_lineterminator( - &mut self, - kind: TokenKind, - routine: Option<&'static str>, - ) -> Result<(), ParseError> { - let next_token = self - .cursor - .next_skip(|tk| tk.kind == TokenKind::LineTerminator) - .ok_or(ParseError::AbruptEnd)?; - - if next_token.kind == kind { - Ok(()) - } else { - Err(ParseError::Expected( - vec![kind], - next_token.clone(), - routine, - )) - } - } - /// Returns an error if the next symbol is not the punctuator `p` /// Consumes the next symbol otherwise fn expect_punc(