tests added for position

src/lib/syntax/ast/pos.rs

@@ -14,21 +14,12 @@ pub struct Position {
 impl Position {
     /// Create a new position, positions are usually created by Tokens..
     ///
+    /// See [Token](struct.token/Token.html) for example usage
+    ///
     /// # Arguments
     ///
     /// * `line_number` - The line number the token starts at
     /// * `column_number` - The column number the token starts at
-    ///
-    /// # Example from Token
-    ///
-    /// ```
-    ///  pub fn new(data: TokenData, line_number: u64, column_number: u64) -> Token {
-    ///     Token {
-    ///         data: data,
-    ///         pos: Position::new(line_number, column_number),
-    ///     }
-    /// }
-    /// ```
     pub fn new(line_number: u64, column_number: u64) -> Position {
         Position {
             line_number: line_number,

src/lib/syntax/lexer.rs

@@ -111,9 +111,9 @@ impl<'a> Lexer<'a> {
     ///
     /// # Example
     ///
-    /// ```
-    /// let buffer = read_to_string("yourSourceCode.js").unwrap();
-    /// let lexer = Lexer::new(&buffer);
+    /// ```rust,no_run
+    /// let buffer = std::fs::read_to_string("yourSourceCode.js").unwrap();
+    /// let lexer = js::syntax::lexer::Lexer::new(&buffer);
     /// ```
     pub fn new(buffer: &'a str) -> Lexer<'a> {
         Lexer {
@@ -192,8 +192,8 @@ impl<'a> Lexer<'a> {
                     }
                 }
             }
-            let ch = self.next()?;
             self.column_number += 1;
+            let ch = self.next()?;
             match ch {
                 '"' | '\'' => {
                     let mut buf = String::new();
@@ -263,7 +263,12 @@ impl<'a> Lexer<'a> {
                             ch => buf.push(ch),
                         }
                     }
+                    let str_length = buf.len() as u64;
                     self.push_token(TokenData::StringLiteral(buf));
+                    // Why +1? Quotation marks are not included,
+                    // So technically it would be +2, (for both " ") but we want to be 1 less
+                    // to compensate for the incrementing at the top
+                    self.column_number += str_length + 1;
                 }
                 '0' => {
                     let mut buf = String::new();
@@ -338,6 +343,8 @@ impl<'a> Lexer<'a> {
                             Err(_) => TokenData::Identifier(buf.clone()),
                         },
                     });
+                    // Move position forward the length of keyword
+                    self.column_number += (buf_compare.len() - 1) as u64;
                 }
                 ';' => self.push_punc(Punctuator::Semicolon),
                 ':' => self.push_punc(Punctuator::Colon),

tests/lexer_test.rs

@@ -21,3 +21,33 @@ fn check_variable_definition_tokens() {
         TokenData::StringLiteral("hello".to_string())
     );
 }
+
+#[test]
+/// Check positions are correct
+fn check_positions() {
+    let s = &String::from("console.log(\"hello world\");");
+    // -------------------123456789
+    let mut lexer = Lexer::new(s);
+    lexer.lex().expect("finished");
+    // The first column is 1 (not zero indexed)
+    assert_eq!(lexer.tokens[0].pos.column_number, 1);
+    assert_eq!(lexer.tokens[0].pos.line_number, 1);
+    // Dot Token starts on line 7
+    assert_eq!(lexer.tokens[1].pos.column_number, 8);
+    assert_eq!(lexer.tokens[1].pos.line_number, 1);
+    // Log Token starts on line 7
+    assert_eq!(lexer.tokens[2].pos.column_number, 9);
+    assert_eq!(lexer.tokens[2].pos.line_number, 1);
+    // Open parenthesis token starts on line 12
+    assert_eq!(lexer.tokens[3].pos.column_number, 12);
+    assert_eq!(lexer.tokens[3].pos.line_number, 1);
+    // String token starts on line 13
+    assert_eq!(lexer.tokens[4].pos.column_number, 13);
+    assert_eq!(lexer.tokens[4].pos.line_number, 1);
+    // Close parenthesis token starts on line 26
+    assert_eq!(lexer.tokens[5].pos.column_number, 26);
+    assert_eq!(lexer.tokens[5].pos.line_number, 1);
+    // Semi Colon token starts on line 27
+    assert_eq!(lexer.tokens[6].pos.column_number, 27);
+    assert_eq!(lexer.tokens[6].pos.line_number, 1);
+}