updates to lexer

src/lib/syntax/ast/token.rs

@@ -4,13 +4,25 @@ use syntax::ast::pos::Position;
 use syntax::ast::punc::Punctuator;
 
 #[derive(Clone, PartialEq)]
+/// Represents a token
 pub struct Token {
     // // The token
     pub data: TokenData,
     pub pos: Position,
 }
 
+impl Token {
+    /// Create a new detailed token from the token data, line number and column number
+    pub fn new(data: TokenData, line_number: u64, column_number: u64) -> Token {
+        Token {
+            data: data,
+            pos: Position::new(line_number, column_number),
+        }
+    }
+}
+
 #[derive(Clone, PartialEq)]
+/// Represents the type of Token
 pub enum TokenData {
     /// A boolean literal, which is either `true` or `false`
     TBooleanLiteral(bool),

src/lib/syntax/lexer.rs

@@ -1,5 +1,6 @@
-use syntax::ast::token::Token;
+use syntax::ast::token::{Token, TokenData};
 
+/// A javascript Lexer
 pub struct Lexer {
     // The list fo tokens generated so far
     pub tokens: Vec<Token>,
@@ -10,3 +11,19 @@ pub struct Lexer {
     // the reader
     buffer: String,
 }
+
+impl Lexer {
+    pub fn new(buffer: String) -> Lexer {
+        Lexer {
+            tokens: Vec::new(),
+            line_number: 1,
+            column_number: 0,
+            buffer: buffer,
+        }
+    }
+
+    fn push_token(&mut self, tk: TokenData) {
+        self.tokens
+            .push(Token::new(tk, self.line_number, self.column_number))
+    }
+}