adding tokenisation

Cargo.toml

@@ -5,6 +5,10 @@ authors = ["Jason Williams <jase.williams@gmail.com>"]
 
 [dependencies]
 
+[lib]
+name = "js"
+path = "src/lib/lib.rs"
+
 [[bin]]
 name = "js"
 path = "src/bin/bin.rs"

src/lib/lib.rs

@@ -0,0 +1 @@
+pub mod syntax;

src/lib/syntax/ast/mod.rs

@@ -0,0 +1 @@
+pub mod token;

src/lib/syntax/ast/token.rs

@@ -0,0 +1,48 @@
+use std::fmt::{Display, Formatter, Result};
+
+#[derive(Clone, PartialEq)]
+pub struct Token {
+    // // The token
+    pub data: TokenData,
+    pub pos: Position,
+}
+
+pub enum TokenData {
+    /// A boolean literal, which is either `true` or `false`
+    TBooleanLiteral(bool),
+    /// The end of the file
+    TEOF,
+    /// An identifier
+    TIdentifier(String),
+    /// A keyword
+    TKeyword(Keyword),
+    /// A `null` literal
+    TNullLiteral,
+    /// A numeric literal
+    TNumericLiteral(f64),
+    /// A piece of punctuation
+    TPunctuator(Punctuator),
+    /// A string literal
+    TStringLiteral(String),
+    /// A regular expression
+    TRegularExpression(String),
+    /// A comment
+    TComment(String),
+}
+
+impl Display for TokenData {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        match self.clone() {
+            TokenData::TBooleanLiteral(val) => write!(f, "{}", val),
+            TEOF => write!(f, "end of file"),
+            TokenData::TIdentifier(ident) => write!(f, "{}", ident),
+            TokenData::TKeyword(word) => write!(f, "{}", word),
+            TNullLiteral => write!(f, "null"),
+            TokenData::TNumericLiteral(num) => write!(f, "{}", num),
+            TokenData::TPunctuator(punc) => write!(f, "{}", punc),
+            TokenData::TStringLiteral(lit) => write!(f, "{}", lit),
+            TokenData::TRegularExpression(reg) => write!(f, "{}", reg),
+            TokenData::TComment(comm) => write!(f, "/*{}*/", comm),
+        }
+    }
+}

src/lib/syntax/lexer.rs

@@ -0,0 +1,12 @@
+use syntax::ast::token::Token;
+
+pub struct Lexer {
+    // The list fo tokens generated so far
+    pub tokens: Vec<Token>,
+    // The current line number in the script
+    line_number: u64,
+    // the current column number in the script
+    column_number: u64,
+    // the reader
+    buffer: String,
+}

src/lib/syntax/mod.rs

@@ -0,0 +1,2 @@
+pub mod ast;
+pub mod lexer;