From a96b3299a01be8359c781c36c265e9e93fe3c397 Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams@gmail.com>
Date: Thu, 9 Aug 2018 18:30:18 +0100
Subject: [PATCH] adding tokenisation

---
 Cargo.toml                  |  6 ++++-
 src/lib/lib.rs              |  1 +
 src/lib/syntax/ast/mod.rs   |  1 +
 src/lib/syntax/ast/token.rs | 48 +++++++++++++++++++++++++++++++++++++
 src/lib/syntax/lexer.rs     | 12 ++++++++++
 src/lib/syntax/mod.rs       |  2 ++
 6 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 src/lib/lib.rs
 create mode 100644 src/lib/syntax/ast/mod.rs
 create mode 100644 src/lib/syntax/ast/token.rs
 create mode 100644 src/lib/syntax/lexer.rs
 create mode 100644 src/lib/syntax/mod.rs

diff --git a/Cargo.toml b/Cargo.toml
index b4c678ed9d..a74d8e04e5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,6 +5,10 @@ authors = ["Jason Williams <jase.williams@gmail.com>"]
 
 [dependencies]
 
+[lib]
+name = "js"
+path = "src/lib/lib.rs"
+
 [[bin]]
 name = "js"
-path = "src/bin/bin.rs"
\ No newline at end of file
+path = "src/bin/bin.rs"
diff --git a/src/lib/lib.rs b/src/lib/lib.rs
new file mode 100644
index 0000000000..4a39d2cfa8
--- /dev/null
+++ b/src/lib/lib.rs
@@ -0,0 +1 @@
+pub mod syntax;
diff --git a/src/lib/syntax/ast/mod.rs b/src/lib/syntax/ast/mod.rs
new file mode 100644
index 0000000000..79c66ba635
--- /dev/null
+++ b/src/lib/syntax/ast/mod.rs
@@ -0,0 +1 @@
+pub mod token;
diff --git a/src/lib/syntax/ast/token.rs b/src/lib/syntax/ast/token.rs
new file mode 100644
index 0000000000..f1af432737
--- /dev/null
+++ b/src/lib/syntax/ast/token.rs
@@ -0,0 +1,48 @@
+use std::fmt::{Display, Formatter, Result};
+
+#[derive(Clone, PartialEq)]
+pub struct Token {
+    // // The token
+    pub data: TokenData,
+    pub pos: Position,
+}
+
+pub enum TokenData {
+    /// A boolean literal, which is either `true` or `false`
+    TBooleanLiteral(bool),
+    /// The end of the file
+    TEOF,
+    /// An identifier
+    TIdentifier(String),
+    /// A keyword
+    TKeyword(Keyword),
+    /// A `null` literal
+    TNullLiteral,
+    /// A numeric literal
+    TNumericLiteral(f64),
+    /// A piece of punctuation
+    TPunctuator(Punctuator),
+    /// A string literal
+    TStringLiteral(String),
+    /// A regular expression
+    TRegularExpression(String),
+    /// A comment
+    TComment(String),
+}
+
+impl Display for TokenData {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        match self.clone() {
+            TokenData::TBooleanLiteral(val) => write!(f, "{}", val),
+            TEOF => write!(f, "end of file"),
+            TokenData::TIdentifier(ident) => write!(f, "{}", ident),
+            TokenData::TKeyword(word) => write!(f, "{}", word),
+            TNullLiteral => write!(f, "null"),
+            TokenData::TNumericLiteral(num) => write!(f, "{}", num),
+            TokenData::TPunctuator(punc) => write!(f, "{}", punc),
+            TokenData::TStringLiteral(lit) => write!(f, "{}", lit),
+            TokenData::TRegularExpression(reg) => write!(f, "{}", reg),
+            TokenData::TComment(comm) => write!(f, "/*{}*/", comm),
+        }
+    }
+}
diff --git a/src/lib/syntax/lexer.rs b/src/lib/syntax/lexer.rs
new file mode 100644
index 0000000000..d73db5f5e4
--- /dev/null
+++ b/src/lib/syntax/lexer.rs
@@ -0,0 +1,12 @@
+use syntax::ast::token::Token;
+
+pub struct Lexer {
+    // The list fo tokens generated so far
+    pub tokens: Vec<Token>,
+    // The current line number in the script
+    line_number: u64,
+    // the current column number in the script
+    column_number: u64,
+    // the reader
+    buffer: String,
+}
diff --git a/src/lib/syntax/mod.rs b/src/lib/syntax/mod.rs
new file mode 100644
index 0000000000..c3f86b1376
--- /dev/null
+++ b/src/lib/syntax/mod.rs
@@ -0,0 +1,2 @@
+pub mod ast;
+pub mod lexer;