alphabetical ordering of punctuation, all keywords added, support for spread operator, new tests for punctuation (WIP)

.vscode/tasks.json

@@ -1,22 +1,29 @@
 {
-    // See https://go.microsoft.com/fwlink/?LinkId=733558 
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "type": "process",
-            "label": "Cargo Run",
-            "command": "cargo",
-            "args": [
-                "run"
-            ],
-            "problemMatcher": [
-                "$rustc"
-            ],
-            "group": {
-                "kind": "build",
-                "isDefault": true
-            }
-        }
-    ]
-}
+  // See https://go.microsoft.com/fwlink/?LinkId=733558
+  // for the documentation about the tasks.json format
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "process",
+      "label": "Cargo Run",
+      "command": "cargo",
+      "args": ["run"],
+      "problemMatcher": ["$rustc"],
+      "group": {
+        "kind": "build",
+        "isDefault": true
+      }
+    },
+    {
+      "type": "process",
+      "label": "Cargo Test",
+      "command": "cargo",
+      "args": ["test"],
+      "problemMatcher": ["$rustc"],
+      "group": {
+        "kind": "test",
+        "isDefault": true
+      }
+    }
+  ]
+}

src/lib/exec.rs

@@ -1,7 +1,7 @@
 use crate::environment::lexical_environment::{new_function_environment, LexicalEnvironment};
 use crate::js::function::{Function, RegularFunction};
 use crate::js::object::{INSTANCE_PROTOTYPE, PROTOTYPE};
-use crate::js::value::{from_value, to_value, ResultValue, Value, ValueData};
+use crate::js::value::{from_value, to_value, ResultValue, ValueData};
 use crate::js::{array, console, function, json, math, object, string};
 use crate::syntax::ast::constant::Const;
 use crate::syntax::ast::expr::{Expr, ExprDef};

src/lib/syntax/ast/keyword.rs

@@ -8,6 +8,8 @@ use std::str::FromStr;
 /// A Javascript Keyword
 /// As specificed by https://www.ecma-international.org/ecma-262/#sec-keywords
 pub enum Keyword {
+    /// The `await` keyword
+    Await,
     /// The `break` keyword
     Break,
     /// The `case` keyword
@@ -32,6 +34,8 @@ pub enum Keyword {
     Else,
     /// The `enum` keyword
     Enum,
+    /// The `export` keyword
+    Export,
     /// The `extends` keyword
     Extends,
     /// The `finally` keyword
@@ -74,6 +78,8 @@ pub enum Keyword {
     While,
     /// The `with` keyword
     With,
+    /// The 'yield' keyword
+    Yield,
 }
 
 #[derive(Debug, Clone)]
@@ -99,6 +105,7 @@ impl FromStr for Keyword {
     type Err = KeywordError;
     fn from_str(s: &str) -> Result<Keyword, Self::Err> {
         match s {
+            "await" => Ok(Await),
             "break" => Ok(Break),
             "case" => Ok(Case),
             "catch" => Ok(Catch),
@@ -112,6 +119,7 @@ impl FromStr for Keyword {
             "else" => Ok(Else),
             "enum" => Ok(Enum),
             "extends" => Ok(Extends),
+            "export" => Ok(Export),
             "finally" => Ok(Finally),
             "for" => Ok(For),
             "function" => Ok(Function),
@@ -132,6 +140,7 @@ impl FromStr for Keyword {
             "void" => Ok(Void),
             "while" => Ok(While),
             "with" => Ok(With),
+            "yield" => Ok(Yield),
             _ => Err(KeywordError),
         }
     }
@@ -142,6 +151,7 @@ impl Display for Keyword {
             f,
             "{}",
             match *self {
+                Await => "await",
                 Break => "break",
                 Case => "case",
                 Catch => "catch",
@@ -155,6 +165,7 @@ impl Display for Keyword {
                 Else => "else",
                 Enum => "enum",
                 Extends => "extends",
+                Export => "export",
                 Finally => "finally",
                 For => "for",
                 Function => "function",
@@ -175,6 +186,7 @@ impl Display for Keyword {
                 Void => "void",
                 While => "while",
                 With => "with",
+                Yield => "yield",
             }
         )
     }

src/lib/syntax/ast/punc.rs

@@ -2,104 +2,106 @@ use std::fmt::{Display, Error, Formatter};
 #[derive(PartialEq, Clone, Debug)]
 /// Punctuation
 pub enum Punctuator {
-    /// `{`
-    OpenBlock,
+    /// `+`
+    Add,
+    /// `&`
+    And,
+    /// `=>`
+    Arrow,
+    /// `=`
+    Assign,
+    /// `+=`
+    AssignAdd,
+    /// `&=`
+    AssignAnd,
+    /// `/=`
+    AssignDiv,
+    /// `<<=`
+    AssignLeftSh,
+    /// `%=`
+    AssignMod,
+    /// `*=`
+    AssignMul,
+    /// `|=`
+    AssignOr,
+    /// `>>>=`
+    AssignRightSh,
+    /// `-=`
+    AssignSub,
+    /// `>>>=`
+    AssignURightSh,
+    /// `^=`
+    AssignXor,
+    /// `&&`
+    BoolAnd,
+    /// `||`
+    BoolOr,
     /// `}`
     CloseBlock,
-    /// `(`
-    OpenParen,
-    /// `)`
-    CloseParen,
-    /// `[`
-    OpenBracket,
     /// `]`
     CloseBracket,
-    /// `.`
-    Dot,
-    /// `;`
-    Semicolon,
+    /// `)`
+    CloseParen,
+    /// `:`
+    Colon,
     /// `,`
     Comma,
-    /// `<`
-    LessThan,
+    /// `--`
+    Dec,
+    /// `/`
+    Div,
+    /// `.`
+    Dot,
+    /// `==`
+    Eq,
     /// `>`
     GreaterThan,
-    /// `<=`
-    LessThanOrEq,
     /// `>=`
     GreaterThanOrEq,
-    /// `==`
-    Eq,
+    /// `++`
+    Inc,
+    /// `<<`
+    LeftSh,
+    /// `<`
+    LessThan,
+    /// `<=`
+    LessThanOrEq,
+    /// `%`
+    Mod,
+    /// `*`
+    Mul,
+    /// `~`
+    Neg,
+    /// `!`
+    Not,
     /// `!=`
     NotEq,
+    /// `{`
+    OpenBlock,
+    /// `[`
+    OpenBracket,
+    /// `(`
+    OpenParen,
+    /// `|`
+    Or,
+    /// `?`
+    Question,
+    /// `>>`
+    RightSh,
+    /// `;`
+    Semicolon,
+    /// `...`
+    Spread,
     /// `===`
     StrictEq,
     /// `!==`
     StrictNotEq,
-    /// `+`
-    Add,
     /// `-`
     Sub,
-    /// `*`
-    Mul,
-    /// `/`
-    Div,
-    /// `%`
-    Mod,
-    /// `++`
-    Inc,
-    /// `--`
-    Dec,
-    /// `<<`
-    LeftSh,
-    /// `>>`
-    RightSh,
     /// `>>>`
     URightSh,
-    /// `&`
-    And,
-    /// `|`
-    Or,
     /// `^`
     Xor,
-    /// `!`
-    Not,
-    /// `~`
-    Neg,
-    /// `&&`
-    BoolAnd,
-    /// `||`
-    BoolOr,
-    /// `?`
-    Question,
-    /// `:`
-    Colon,
-    /// `=`
-    Assign,
-    /// `+=`
-    AssignAdd,
-    /// `-=`
-    AssignSub,
-    /// `*=`
-    AssignMul,
-    /// `/=`
-    AssignDiv,
-    /// `%=`
-    AssignMod,
-    /// `<<=`
-    AssignLeftSh,
-    /// `>>=`
-    AssignRightSh,
-    /// `>>>=`
-    AssignURightSh,
-    /// `&=`
-    AssignAnd,
-    /// `|=`
-    AssignOr,
-    /// `^=`
-    AssignXor,
-    /// `=>`
-    Arrow,
 }
 impl Display for Punctuator {
     fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
@@ -107,55 +109,56 @@ impl Display for Punctuator {
             f,
             "{}",
             match self {
-                Punctuator::OpenBlock => "{",
+                Punctuator::Add => "+",
+                Punctuator::And => "&",
+                Punctuator::Arrow => "=>",
+                Punctuator::Assign => "=",
+                Punctuator::AssignAdd => "+=",
+                Punctuator::AssignAnd => "&=",
+                Punctuator::AssignDiv => "/=",
+                Punctuator::AssignLeftSh => "<<=",
+                Punctuator::AssignMod => "%=",
+                Punctuator::AssignMul => "*=",
+                Punctuator::AssignOr => "|=",
+                Punctuator::AssignRightSh => ">>=",
+                Punctuator::AssignSub => "-=",
+                Punctuator::AssignURightSh => ">>>=",
+                Punctuator::AssignXor => "^=",
+                Punctuator::BoolAnd => "&&",
+                Punctuator::BoolOr => "||",
                 Punctuator::CloseBlock => "}",
-                Punctuator::OpenParen => "(",
-                Punctuator::CloseParen => ")",
-                Punctuator::OpenBracket => "[",
                 Punctuator::CloseBracket => "]",
-                Punctuator::Dot => ".",
-                Punctuator::Semicolon => ";",
+                Punctuator::CloseParen => ")",
+                Punctuator::Colon => ":",
                 Punctuator::Comma => ",",
-                Punctuator::LessThan => "<",
+                Punctuator::Dec => "--",
+                Punctuator::Div => "/",
+                Punctuator::Dot => ".",
+                Punctuator::Eq => "==",
                 Punctuator::GreaterThan => ">",
-                Punctuator::LessThanOrEq => "<=",
                 Punctuator::GreaterThanOrEq => ">=",
-                Punctuator::Eq => "==",
+                Punctuator::Inc => "++",
+                Punctuator::LeftSh => "<<",
+                Punctuator::LessThan => "<",
+                Punctuator::LessThanOrEq => "<=",
+                Punctuator::Mod => "%",
+                Punctuator::Mul => "*",
+                Punctuator::Neg => "~",
+                Punctuator::Not => "!",
                 Punctuator::NotEq => "!=",
+                Punctuator::OpenBlock => "{",
+                Punctuator::OpenBracket => "[",
+                Punctuator::OpenParen => "(",
+                Punctuator::Or => "|",
+                Punctuator::Question => "?",
+                Punctuator::RightSh => ">>",
+                Punctuator::Semicolon => ";",
+                Punctuator::Spread => "...",
                 Punctuator::StrictEq => "===",
                 Punctuator::StrictNotEq => "!==",
-                Punctuator::Add => "+",
                 Punctuator::Sub => "-",
-                Punctuator::Mul => "*",
-                Punctuator::Div => "/",
-                Punctuator::Mod => "%",
-                Punctuator::Inc => "++",
-                Punctuator::Dec => "--",
-                Punctuator::LeftSh => "<<",
-                Punctuator::RightSh => ">>",
                 Punctuator::URightSh => ">>>",
-                Punctuator::And => "&",
-                Punctuator::Or => "|",
                 Punctuator::Xor => "^",
-                Punctuator::Not => "!",
-                Punctuator::Neg => "~",
-                Punctuator::BoolAnd => "&&",
-                Punctuator::BoolOr => "||",
-                Punctuator::Question => "?",
-                Punctuator::Colon => ":",
-                Punctuator::Assign => "=",
-                Punctuator::AssignAdd => "+=",
-                Punctuator::AssignSub => "-=",
-                Punctuator::AssignMul => "*=",
-                Punctuator::AssignDiv => "/=",
-                Punctuator::AssignMod => "%=",
-                Punctuator::AssignLeftSh => "<<=",
-                Punctuator::AssignRightSh => ">>=",
-                Punctuator::AssignURightSh => ">>>=",
-                Punctuator::AssignAnd => "&=",
-                Punctuator::AssignOr => "|=",
-                Punctuator::AssignXor => "^=",
-                Punctuator::Arrow => "=>",
             }
         )
     }

src/lib/syntax/lexer.rs

@@ -436,7 +436,18 @@ impl<'a> Lexer<'a> {
                 }
                 ';' => self.push_punc(Punctuator::Semicolon),
                 ':' => self.push_punc(Punctuator::Colon),
-                '.' => self.push_punc(Punctuator::Dot),
+                '.' => {
+                    // . or ...
+                    if self.next_is('.') {
+                        if self.next_is('.') {
+                            self.push_punc(Punctuator::Spread);
+                        } else {
+                            return Err(LexerError::new("Expecting Token ."));
+                        }
+                    } else {
+                        self.push_punc(Punctuator::Dot);
+                    };
+                }
                 '(' => self.push_punc(Punctuator::OpenParen),
                 ')' => self.push_punc(Punctuator::CloseParen),
                 ',' => self.push_punc(Punctuator::Comma),
@@ -543,11 +554,172 @@ mod tests {
     use super::*;
     use crate::syntax::ast::keyword::Keyword;
 
+    #[test]
+    fn check_string() {
+        let s = "'aaa' \"bbb\"";
+        let mut lexer = Lexer::new(s);
+        lexer.lex().unwrap();
+        assert_eq!(
+            lexer.tokens[0].data,
+            TokenData::StringLiteral("aaa".to_string())
+        );
+
+        assert_eq!(
+            lexer.tokens[1].data,
+            TokenData::StringLiteral("bbb".to_string())
+        );
+    }
+
+    #[test]
+    fn check_punctuators() {
+        // TODO: Support ** ++
+        // https://tc39.es/ecma262/#sec-punctuators
+        let s = "{ ( ) [ ] . ... ; , < > <= >= == != === !== \
+                 + - * % -- << >> >>> & | ^ ! ~ && || ? : \
+                 = += -= *= &= **= <<= >>= >>>= &= |= ^= =>";
+        let mut lexer = Lexer::new(s);
+        lexer.lex().unwrap();
+        assert_eq!(
+            lexer.tokens[0].data,
+            TokenData::Punctuator(Punctuator::OpenBlock)
+        );
+        assert_eq!(
+            lexer.tokens[1].data,
+            TokenData::Punctuator(Punctuator::OpenParen)
+        );
+        assert_eq!(
+            lexer.tokens[2].data,
+            TokenData::Punctuator(Punctuator::CloseParen)
+        );
+        assert_eq!(
+            lexer.tokens[3].data,
+            TokenData::Punctuator(Punctuator::OpenBracket)
+        );
+        assert_eq!(
+            lexer.tokens[4].data,
+            TokenData::Punctuator(Punctuator::CloseBracket)
+        );
+        assert_eq!(lexer.tokens[5].data, TokenData::Punctuator(Punctuator::Dot));
+        assert_eq!(
+            lexer.tokens[6].data,
+            TokenData::Punctuator(Punctuator::Spread)
+        );
+        assert_eq!(
+            lexer.tokens[7].data,
+            TokenData::Punctuator(Punctuator::Semicolon)
+        );
+        assert_eq!(
+            lexer.tokens[8].data,
+            TokenData::Punctuator(Punctuator::Comma)
+        );
+        assert_eq!(
+            lexer.tokens[9].data,
+            TokenData::Punctuator(Punctuator::LessThan)
+        );
+        assert_eq!(
+            lexer.tokens[10].data,
+            TokenData::Punctuator(Punctuator::GreaterThan)
+        );
+        assert_eq!(
+            lexer.tokens[11].data,
+            TokenData::Punctuator(Punctuator::LessThanOrEq)
+        );
+        assert_eq!(
+            lexer.tokens[12].data,
+            TokenData::Punctuator(Punctuator::GreaterThanOrEq)
+        );
+        assert_eq!(lexer.tokens[13].data, TokenData::Punctuator(Punctuator::Eq));
+        assert_eq!(
+            lexer.tokens[14].data,
+            TokenData::Punctuator(Punctuator::NotEq)
+        );
+        assert_eq!(
+            lexer.tokens[15].data,
+            TokenData::Punctuator(Punctuator::StrictEq)
+        );
+        assert_eq!(
+            lexer.tokens[16].data,
+            TokenData::Punctuator(Punctuator::StrictNotEq)
+        );
+        assert_eq!(
+            lexer.tokens[17].data,
+            TokenData::Punctuator(Punctuator::Add)
+        );
+        assert_eq!(
+            lexer.tokens[18].data,
+            TokenData::Punctuator(Punctuator::Sub)
+        );
+        assert_eq!(
+            lexer.tokens[19].data,
+            TokenData::Punctuator(Punctuator::Mul)
+        );
+        assert_eq!(
+            lexer.tokens[20].data,
+            TokenData::Punctuator(Punctuator::Mod)
+        );
+        assert_eq!(
+            lexer.tokens[21].data,
+            TokenData::Punctuator(Punctuator::Dec)
+        );
+        assert_eq!(
+            lexer.tokens[22].data,
+            TokenData::Punctuator(Punctuator::LeftSh)
+        );
+    }
+
+    #[test]
+    fn check_keywords() {
+        // https://tc39.es/ecma262/#sec-keywords
+        let s = "await break case catch class const continue debugger default delete \
+                 do else export extends finally for function if import in instanceof \
+                 new return super switch this throw try typeof var void while with yield";
+
+        let mut lexer = Lexer::new(s);
+        lexer.lex().unwrap();
+        assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Await));
+        assert_eq!(lexer.tokens[1].data, TokenData::Keyword(Keyword::Break));
+        assert_eq!(lexer.tokens[2].data, TokenData::Keyword(Keyword::Case));
+        assert_eq!(lexer.tokens[3].data, TokenData::Keyword(Keyword::Catch));
+        assert_eq!(lexer.tokens[4].data, TokenData::Keyword(Keyword::Class));
+        assert_eq!(lexer.tokens[5].data, TokenData::Keyword(Keyword::Const));
+        assert_eq!(lexer.tokens[6].data, TokenData::Keyword(Keyword::Continue));
+        assert_eq!(lexer.tokens[7].data, TokenData::Keyword(Keyword::Debugger));
+        assert_eq!(lexer.tokens[8].data, TokenData::Keyword(Keyword::Default));
+        assert_eq!(lexer.tokens[9].data, TokenData::Keyword(Keyword::Delete));
+        assert_eq!(lexer.tokens[10].data, TokenData::Keyword(Keyword::Do));
+        assert_eq!(lexer.tokens[11].data, TokenData::Keyword(Keyword::Else));
+        assert_eq!(lexer.tokens[12].data, TokenData::Keyword(Keyword::Export));
+        assert_eq!(lexer.tokens[13].data, TokenData::Keyword(Keyword::Extends));
+        assert_eq!(lexer.tokens[14].data, TokenData::Keyword(Keyword::Finally));
+        assert_eq!(lexer.tokens[15].data, TokenData::Keyword(Keyword::For));
+        assert_eq!(lexer.tokens[16].data, TokenData::Keyword(Keyword::Function));
+        assert_eq!(lexer.tokens[17].data, TokenData::Keyword(Keyword::If));
+        assert_eq!(lexer.tokens[18].data, TokenData::Keyword(Keyword::Import));
+        assert_eq!(lexer.tokens[19].data, TokenData::Keyword(Keyword::In));
+        assert_eq!(
+            lexer.tokens[20].data,
+            TokenData::Keyword(Keyword::InstanceOf)
+        );
+        assert_eq!(lexer.tokens[21].data, TokenData::Keyword(Keyword::New));
+        assert_eq!(lexer.tokens[22].data, TokenData::Keyword(Keyword::Return));
+        assert_eq!(lexer.tokens[23].data, TokenData::Keyword(Keyword::Super));
+        assert_eq!(lexer.tokens[24].data, TokenData::Keyword(Keyword::Switch));
+        assert_eq!(lexer.tokens[25].data, TokenData::Keyword(Keyword::This));
+        assert_eq!(lexer.tokens[26].data, TokenData::Keyword(Keyword::Throw));
+        assert_eq!(lexer.tokens[27].data, TokenData::Keyword(Keyword::Try));
+        assert_eq!(lexer.tokens[28].data, TokenData::Keyword(Keyword::TypeOf));
+        assert_eq!(lexer.tokens[29].data, TokenData::Keyword(Keyword::Var));
+        assert_eq!(lexer.tokens[30].data, TokenData::Keyword(Keyword::Void));
+        assert_eq!(lexer.tokens[31].data, TokenData::Keyword(Keyword::While));
+        assert_eq!(lexer.tokens[32].data, TokenData::Keyword(Keyword::With));
+        assert_eq!(lexer.tokens[33].data, TokenData::Keyword(Keyword::Yield));
+    }
+
     #[test]
     fn check_variable_definition_tokens() {
         let s = &String::from("let a = 'hello';");
         let mut lexer = Lexer::new(s);
-        lexer.lex().expect("finished");
+        lexer.lex().unwrap();
         assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Let));
         assert_eq!(lexer.tokens[1].data, TokenData::Identifier("a".to_string()));
         assert_eq!(
@@ -565,7 +737,7 @@ mod tests {
         let s = &String::from("console.log(\"hello world\");");
         // -------------------123456789
         let mut lexer = Lexer::new(s);
-        lexer.lex().expect("finished");
+        lexer.lex().unwrap();
         // The first column is 1 (not zero indexed)
         assert_eq!(lexer.tokens[0].pos.column_number, 1);
         assert_eq!(lexer.tokens[0].pos.line_number, 1);
@@ -595,7 +767,7 @@ mod tests {
         // Here we want an example of decrementing an integer
         let s = &String::from("let a = b--;");
         let mut lexer = Lexer::new(s);
-        lexer.lex().expect("finished");
+        lexer.lex().unwrap();
         assert_eq!(lexer.tokens[4].data, TokenData::Punctuator(Punctuator::Dec));
         // Decrementing means adding 2 characters '--', the lexer should consume it as a single token
         // and move the curser forward by 2, meaning the next token should be a semicolon
@@ -608,7 +780,7 @@ mod tests {
     #[test]
     fn numbers() {
         let mut lexer = Lexer::new("1 2 0x34 056 7.89 5e3 5e+3 5e-3 0b10 0O123 0999");
-        lexer.lex().expect("finished");
+        lexer.lex().unwrap();
         assert_eq!(lexer.tokens[0].data, TokenData::NumericLiteral(1.0));
         assert_eq!(lexer.tokens[1].data, TokenData::NumericLiteral(2.0));
         assert_eq!(lexer.tokens[2].data, TokenData::NumericLiteral(52.0));
@@ -625,7 +797,7 @@ mod tests {
     #[test]
     fn test_single_number_without_semicolon() {
         let mut lexer = Lexer::new("1");
-        lexer.lex().expect("finished");
+        lexer.lex().unwrap();
         dbg!(lexer.tokens);
     }