adding Let

src/bin/bin.rs

@@ -5,5 +5,6 @@ use std::fs::read_to_string;
 pub fn main() {
     let buffer = read_to_string("test.js").unwrap();
     let mut lexer = Lexer::new(&buffer);
-    lexer.lex().expect("finished")
+    lexer.lex().expect("finished");
+    println!("{:?}", lexer.tokens);
 }

src/lib/syntax/ast/keyword.rs

@@ -4,71 +4,73 @@ use std::fmt::{Display, Formatter};
 use std::str::FromStr;
 use syntax::ast::keyword::Keyword::*;
 
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Debug)]
 /// A Javascript Keyword
 pub enum Keyword {
     /// The `break` keyword
-    KBreak,
+    Break,
     /// The `case` keyword
-    KCase,
+    Case,
     /// The `catch` keyword
-    KCatch,
+    Catch,
     /// The `class` keyword, which is reserved for future use
-    KClass,
+    Class,
     /// The `continue` keyword
-    KContinue,
+    Continue,
     /// The `debugger` keyword
-    KDebugger,
+    Debugger,
     /// The `default` keyword
-    KDefault,
+    Default,
     /// The `delete` keyword
-    KDelete,
+    Delete,
     /// The `do` keyword
-    KDo,
+    Do,
     /// The `else` keyword
-    KElse,
+    Else,
     /// The `enum` keyword
-    KEnum,
+    Enum,
     /// The `extends` keyword
-    KExtends,
+    Extends,
     /// The `finally` keyword
-    KFinally,
+    Finally,
     /// The `for` keyword
-    KFor,
+    For,
     /// The `function` keyword
-    KFunction,
+    Function,
     /// The `if` keyword
-    KIf,
+    If,
     /// The `in` keyword
-    KIn,
+    In,
     /// The `instanceof` keyword
-    KInstanceOf,
+    InstanceOf,
     /// The `import` keyword
-    KImport,
+    Import,
+    /// The `let` keyword
+    Let,
     /// The `new` keyword
-    KNew,
+    New,
     /// The `return` keyword
-    KReturn,
+    Return,
     /// The `super` keyword
-    KSuper,
+    Super,
     /// The `switch` keyword
-    KSwitch,
+    Switch,
     /// The `this` keyword
-    KThis,
+    This,
     /// The `throw` keyword
-    KThrow,
+    Throw,
     /// The `try` keyword
-    KTry,
+    Try,
     /// The `typeof` keyword
-    KTypeOf,
+    TypeOf,
     /// The `var` keyword
-    KVar,
+    Var,
     /// The `void` keyword
-    KVoid,
+    Void,
     /// The `while` keyword
-    KWhile,
+    While,
     /// The `with` keyword
-    KWith,
+    With,
 }
 
 #[derive(Debug, Clone)]
@@ -94,37 +96,38 @@ impl FromStr for Keyword {
     type Err = KeywordError;
     fn from_str(s: &str) -> Result<Keyword, Self::Err> {
         match s {
-            "break" => Ok(KBreak),
+            "break" => Ok(Break),
-            "case" => Ok(KCase),
+            "case" => Ok(Case),
-            "catch" => Ok(KCatch),
+            "catch" => Ok(Catch),
-            "class" => Ok(KClass),
+            "class" => Ok(Class),
-            "continue" => Ok(KContinue),
+            "continue" => Ok(Continue),
-            "debugger" => Ok(KDebugger),
+            "debugger" => Ok(Debugger),
-            "default" => Ok(KDefault),
+            "default" => Ok(Default),
-            "delete" => Ok(KDelete),
+            "delete" => Ok(Delete),
-            "do" => Ok(KDo),
+            "do" => Ok(Do),
-            "else" => Ok(KElse),
+            "else" => Ok(Else),
-            "enum" => Ok(KEnum),
+            "enum" => Ok(Enum),
-            "extends" => Ok(KExtends),
+            "extends" => Ok(Extends),
-            "finally" => Ok(KFinally),
+            "finally" => Ok(Finally),
-            "for" => Ok(KFor),
+            "for" => Ok(For),
-            "function" => Ok(KFunction),
+            "function" => Ok(Function),
-            "if" => Ok(KIf),
+            "if" => Ok(If),
-            "in" => Ok(KIn),
+            "in" => Ok(In),
-            "instanceof" => Ok(KInstanceOf),
+            "instanceof" => Ok(InstanceOf),
-            "import" => Ok(KImport),
+            "import" => Ok(Import),
-            "new" => Ok(KNew),
+            "let" => Ok(Let),
-            "return" => Ok(KReturn),
+            "new" => Ok(New),
-            "super" => Ok(KSuper),
+            "return" => Ok(Return),
-            "switch" => Ok(KSwitch),
+            "super" => Ok(Super),
-            "this" => Ok(KThis),
+            "switch" => Ok(Switch),
-            "throw" => Ok(KThrow),
+            "this" => Ok(This),
-            "try" => Ok(KTry),
+            "throw" => Ok(Throw),
-            "typeof" => Ok(KTypeOf),
+            "try" => Ok(Try),
-            "var" => Ok(KVar),
+            "typeof" => Ok(TypeOf),
-            "void" => Ok(KVoid),
+            "var" => Ok(Var),
-            "while" => Ok(KWhile),
+            "void" => Ok(Void),
-            "with" => Ok(KWith),
+            "while" => Ok(While),
+            "with" => Ok(With),
             _ => Err(KeywordError),
         }
     }
@@ -135,37 +138,38 @@ impl Display for Keyword {
             f,
             "{}",
             match *self {
-                KBreak => "break",
+                Break => "break",
-                KCase => "case",
+                Case => "case",
-                KCatch => "catch",
+                Catch => "catch",
-                KClass => "class",
+                Class => "class",
-                KContinue => "continue",
+                Continue => "continue",
-                KDebugger => "debugger",
+                Debugger => "debugger",
-                KDefault => "default",
+                Default => "default",
-                KDelete => "delete",
+                Delete => "delete",
-                KDo => "do",
+                Do => "do",
-                KElse => "else",
+                Else => "else",
-                KEnum => "enum",
+                Enum => "enum",
-                KExtends => "extends",
+                Extends => "extends",
-                KFinally => "finally",
+                Finally => "finally",
-                KFor => "for",
+                For => "for",
-                KFunction => "function",
+                Function => "function",
-                KIf => "if",
+                If => "if",
-                KIn => "in",
+                In => "in",
-                KInstanceOf => "instanceof",
+                InstanceOf => "instanceof",
-                KImport => "import",
+                Import => "import",
-                KNew => "new",
+                Let => "let",
-                KReturn => "return",
+                New => "new",
-                KSuper => "super",
+                Return => "return",
-                KSwitch => "switch",
+                Super => "super",
-                KThis => "this",
+                Switch => "switch",
-                KThrow => "throw",
+                This => "this",
-                KTry => "try",
+                Throw => "throw",
-                KTypeOf => "typeof",
+                Try => "try",
-                KVar => "var",
+                TypeOf => "typeof",
-                KVoid => "void",
+                Var => "var",
-                KWhile => "while",
+                Void => "void",
-                KWith => "with",
+                While => "while",
+                With => "with",
             }
         )
     }

src/lib/syntax/ast/pos.rs

@@ -1,4 +1,4 @@
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Debug)]
 // A position in Javascript source code
 pub struct Position {
     // Column number

src/lib/syntax/ast/punc.rs

@@ -1,5 +1,5 @@
 use std::fmt::{Display, Error, Formatter};
-#[derive(PartialEq, Clone)]
+#[derive(PartialEq, Clone, Debug)]
 /// Punctuation
 pub enum Punctuator {
     /// `{`

src/lib/syntax/ast/token.rs

@@ -1,13 +1,15 @@
-use std::fmt::{Display, Formatter, Result};
+use std::fmt::{Debug, Display, Formatter, Result};
 use syntax::ast::keyword::Keyword;
 use syntax::ast::pos::Position;
 use syntax::ast::punc::Punctuator;
 
 #[derive(Clone, PartialEq)]
 /// Represents a token
+#[derive(Debug)]
 pub struct Token {
-    // // The token
+    /// The token Data
     pub data: TokenData,
+    /// Token position from original source code
     pub pos: Position,
 }
 
@@ -21,7 +23,25 @@ impl Token {
     }
 }
 
-#[derive(Clone, PartialEq)]
+impl Display for Token {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        write!(f, "{}", self.data)
+    }
+}
+
+pub struct VecToken(Vec<Token>);
+
+impl Debug for VecToken {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        let mut buffer = String::new();
+        for token in &self.0 {
+            buffer.push_str(&token.to_string());
+        }
+        write!(f, "{}", buffer)
+    }
+}
+
+#[derive(Clone, PartialEq, Debug)]
 /// Represents the type of Token
 pub enum TokenData {
     /// A boolean literal, which is either `true` or `false`

src/lib/syntax/lexer.rs

@@ -167,10 +167,12 @@ impl<'a> Lexer<'a> {
             // Check if we've reached the end
             match self.preview_next() {
                 Ok(_) => (), // If there are still characters, carry on
-                Err(LexerError { details }) => {
+                Err(e) => {
-                    if details == "finished" {
+                    if e.details == "finished" {
                         // If there are no more characters left in the Chars iterator, we should just return
                         return Ok(());
+                    } else {
+                        return Err(e);
                     }
                 }
             }

test.js

@@ -1 +1 @@
-console.log('hello world from js');
+let a = (2 * 2)