diff --git a/src/lib/syntax/ast/pos.rs b/src/lib/syntax/ast/pos.rs index 9d4d2a95d8..9f41c55089 100644 --- a/src/lib/syntax/ast/pos.rs +++ b/src/lib/syntax/ast/pos.rs @@ -14,21 +14,12 @@ pub struct Position { impl Position { /// Create a new position, positions are usually created by Tokens.. /// + /// See [Token](struct.token/Token.html) for example usage + /// /// # Arguments /// /// * `line_number` - The line number the token starts at /// * `column_number` - The column number the token starts at - /// - /// # Example from Token - /// - /// ``` - /// pub fn new(data: TokenData, line_number: u64, column_number: u64) -> Token { - /// Token { - /// data: data, - /// pos: Position::new(line_number, column_number), - /// } - /// } - /// ``` pub fn new(line_number: u64, column_number: u64) -> Position { Position { line_number: line_number, diff --git a/src/lib/syntax/lexer.rs b/src/lib/syntax/lexer.rs index 205573debd..1205e44765 100644 --- a/src/lib/syntax/lexer.rs +++ b/src/lib/syntax/lexer.rs @@ -111,9 +111,9 @@ impl<'a> Lexer<'a> { /// /// # Example /// - /// ``` - /// let buffer = read_to_string("yourSourceCode.js").unwrap(); - /// let lexer = Lexer::new(&buffer); + /// ```rust,no_run + /// let buffer = std::fs::read_to_string("yourSourceCode.js").unwrap(); + /// let lexer = js::syntax::lexer::Lexer::new(&buffer); /// ``` pub fn new(buffer: &'a str) -> Lexer<'a> { Lexer { @@ -192,8 +192,8 @@ impl<'a> Lexer<'a> { } } } - let ch = self.next()?; self.column_number += 1; + let ch = self.next()?; match ch { '"' | '\'' => { let mut buf = String::new(); @@ -263,7 +263,12 @@ impl<'a> Lexer<'a> { ch => buf.push(ch), } } + let str_length = buf.len() as u64; self.push_token(TokenData::StringLiteral(buf)); + // Why +1? Quotation marks are not included, + // So technically it would be +2, (for both " ") but we want to be 1 less + // to compensate for the incrementing at the top + self.column_number += str_length + 1; } '0' => { let mut buf = String::new(); @@ -338,6 +343,8 @@ impl<'a> Lexer<'a> { Err(_) => TokenData::Identifier(buf.clone()), }, }); + // Move position forward the length of keyword + self.column_number += (buf_compare.len() - 1) as u64; } ';' => self.push_punc(Punctuator::Semicolon), ':' => self.push_punc(Punctuator::Colon), diff --git a/tests/lexer_test.rs b/tests/lexer_test.rs index 7274220a07..4e980b0b3f 100644 --- a/tests/lexer_test.rs +++ b/tests/lexer_test.rs @@ -21,3 +21,33 @@ fn check_variable_definition_tokens() { TokenData::StringLiteral("hello".to_string()) ); } + +#[test] +/// Check positions are correct +fn check_positions() { + let s = &String::from("console.log(\"hello world\");"); + // -------------------123456789 + let mut lexer = Lexer::new(s); + lexer.lex().expect("finished"); + // The first column is 1 (not zero indexed) + assert_eq!(lexer.tokens[0].pos.column_number, 1); + assert_eq!(lexer.tokens[0].pos.line_number, 1); + // Dot Token starts on line 7 + assert_eq!(lexer.tokens[1].pos.column_number, 8); + assert_eq!(lexer.tokens[1].pos.line_number, 1); + // Log Token starts on line 7 + assert_eq!(lexer.tokens[2].pos.column_number, 9); + assert_eq!(lexer.tokens[2].pos.line_number, 1); + // Open parenthesis token starts on line 12 + assert_eq!(lexer.tokens[3].pos.column_number, 12); + assert_eq!(lexer.tokens[3].pos.line_number, 1); + // String token starts on line 13 + assert_eq!(lexer.tokens[4].pos.column_number, 13); + assert_eq!(lexer.tokens[4].pos.line_number, 1); + // Close parenthesis token starts on line 26 + assert_eq!(lexer.tokens[5].pos.column_number, 26); + assert_eq!(lexer.tokens[5].pos.line_number, 1); + // Semi Colon token starts on line 27 + assert_eq!(lexer.tokens[6].pos.column_number, 27); + assert_eq!(lexer.tokens[6].pos.line_number, 1); +}