Fixed positions in regexes and strict operators. (#295)

I also removed an unused function in the parser and added a test for #294, currently ignored.

boa/src/lib.rs

@@ -50,7 +50,6 @@ fn parser_expr(src: &str) -> Result<Node, String> {
     let mut lexer = Lexer::new(src);
     lexer.lex().map_err(|e| format!("SyntaxError: {}", e))?;
     let tokens = lexer.tokens;
-    // dbg!(&tokens);
     Parser::new(&tokens)
         .parse_all()
         .map_err(|e| format!("ParsingError: {}", e))

boa/src/syntax/lexer/mod.rs

@@ -23,6 +23,7 @@ macro_rules! vop {
         match preview {
             '=' => {
                 $this.next();
+                $this.column_number += 1;
                 $assign_op
             }
             _ => $op,
@@ -33,10 +34,12 @@ macro_rules! vop {
         match preview {
             '=' => {
                 $this.next();
+                $this.column_number += 1;
                 $assign_op
             },
             $($case => {
                 $this.next();
+                $this.column_number += 1;
                 $block
             })+,
             _ => $op
@@ -47,6 +50,7 @@ macro_rules! vop {
         match preview {
             $($case => {
                 $this.next()?;
+                $this.column_number += 1;
                 $block
             })+,
             _ => $op
@@ -135,6 +139,7 @@ impl<'a> Lexer<'a> {
             buffer: buffer.chars().peekable(),
         }
     }
+
     /// Push tokens onto the token queue
     fn push_token(&mut self, tk: TokenKind) {
         self.tokens
@@ -148,16 +153,17 @@ impl<'a> Lexer<'a> {
 
     /// next fetches the next token and return it, or a LexerError if there are no more.
     fn next(&mut self) -> char {
-        match self.buffer.next() {
+        self.buffer.next().expect(
-            Some(ch) => ch,
+            "No more more characters to consume from input stream, \
-            None => panic!("No more more characters to consume from input stream, use preview_next() first to check before calling next()"),
+             use preview_next() first to check before calling next()",
-        }
+        )
     }
 
     /// Preview the next character but don't actually increment
     fn preview_next(&mut self) -> Option<char> {
         self.buffer.peek().copied()
     }
+
     /// Preview a char x indexes further in buf, without incrementing
     fn preview_multiple_next(&mut self, nb_next: usize) -> Option<char> {
         let mut next_peek = None;
@@ -189,7 +195,7 @@ impl<'a> Lexer<'a> {
         Ok(s)
     }
 
-    /// next_is compares the character passed in to the next character, if they match true is returned and the buffer is incremented
+    /// Compares the character passed in to the next character, if they match true is returned and the buffer is incremented
     fn next_is(&mut self, peek: char) -> bool {
         let result = self.preview_next() == Some(peek);
         if result {
@@ -563,6 +569,7 @@ impl<'a> Lexer<'a> {
                                 let mut body = String::new();
                                 let mut regex = false;
                                 loop {
+                                    self.column_number +=1;
                                     match self.buffer.next() {
                                         // end of body
                                         Some('/') => {
@@ -570,8 +577,18 @@ impl<'a> Lexer<'a> {
                                             break;
                                         }
                                         // newline/eof not allowed in regex literal
-                                        Some('\n') | Some('\r') | Some('\u{2028}')
+                                        n @ Some('\n') | n @ Some('\r') | n @ Some('\u{2028}')
-                                        | Some('\u{2029}') | None => break,
+                                        | n @ Some('\u{2029}') => {
+                                            self.column_number = 0;
+                                            if n != Some('\r') {
+                                                self.line_number += 1;
+                                            }
+                                            break
+                                        },
+                                        None => {
+                                            self.column_number -= 1;
+                                            break
+                                        }
                                         // escape sequence
                                         Some('\\') => {
                                             body.push('\\');

boa/src/syntax/lexer/tests.rs

@@ -330,6 +330,18 @@ fn check_positions() {
     assert_eq!(lexer.tokens[6].pos.line_number, 1);
 }
 
+#[test]
+#[ignore]
+fn test_two_divisions_in_expression() {
+    let s = "    return a !== 0 || 1 / a === 1 / b;";
+    let mut lexer = Lexer::new(s);
+    lexer.lex().expect("failed to lex");
+    dbg!(&lexer.tokens);
+
+    assert_eq!(lexer.tokens[11].pos.column_number, 37);
+    assert_eq!(lexer.tokens[11].pos.line_number, 1);
+}
+
 #[test]
 fn check_line_numbers() {
     let s = "x\ny\n";

boa/src/syntax/parser/mod.rs

@@ -222,28 +222,6 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Returns an error if the next symbol is not `tk`
-    fn expect_no_lineterminator(
-        &mut self,
-        kind: TokenKind,
-        routine: Option<&'static str>,
-    ) -> Result<(), ParseError> {
-        let next_token = self
-            .cursor
-            .next_skip(|tk| tk.kind == TokenKind::LineTerminator)
-            .ok_or(ParseError::AbruptEnd)?;
-
-        if next_token.kind == kind {
-            Ok(())
-        } else {
-            Err(ParseError::Expected(
-                vec![kind],
-                next_token.clone(),
-                routine,
-            ))
-        }
-    }
-
     /// Returns an error if the next symbol is not the punctuator `p`
     /// Consumes the next symbol otherwise
     fn expect_punc(