From 383057c7aa9c8c02677fd1db2ff20432867eb637 Mon Sep 17 00:00:00 2001 From: Jason Williams Date: Sat, 8 Jun 2019 18:22:38 +0100 Subject: [PATCH] alphabetical ordering of punctuation, all keywords added, support for spread operator, new tests for punctuation (WIP) --- .vscode/tasks.json | 49 ++++---- src/lib/exec.rs | 2 +- src/lib/syntax/ast/keyword.rs | 12 ++ src/lib/syntax/ast/punc.rs | 231 +++++++++++++++++----------------- src/lib/syntax/lexer.rs | 184 ++++++++++++++++++++++++++- 5 files changed, 336 insertions(+), 142 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index cf65c09b53..f40039aa0b 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,22 +1,29 @@ { - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "2.0.0", - "tasks": [ - { - "type": "process", - "label": "Cargo Run", - "command": "cargo", - "args": [ - "run" - ], - "problemMatcher": [ - "$rustc" - ], - "group": { - "kind": "build", - "isDefault": true - } - } - ] -} \ No newline at end of file + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "type": "process", + "label": "Cargo Run", + "command": "cargo", + "args": ["run"], + "problemMatcher": ["$rustc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "type": "process", + "label": "Cargo Test", + "command": "cargo", + "args": ["test"], + "problemMatcher": ["$rustc"], + "group": { + "kind": "test", + "isDefault": true + } + } + ] +} diff --git a/src/lib/exec.rs b/src/lib/exec.rs index 886bb9ef24..5742b3f56e 100644 --- a/src/lib/exec.rs +++ b/src/lib/exec.rs @@ -1,7 +1,7 @@ use crate::environment::lexical_environment::{new_function_environment, LexicalEnvironment}; use crate::js::function::{Function, RegularFunction}; use crate::js::object::{INSTANCE_PROTOTYPE, PROTOTYPE}; -use crate::js::value::{from_value, to_value, ResultValue, Value, ValueData}; +use crate::js::value::{from_value, to_value, ResultValue, ValueData}; use crate::js::{array, console, function, json, math, object, string}; use crate::syntax::ast::constant::Const; use crate::syntax::ast::expr::{Expr, ExprDef}; diff --git a/src/lib/syntax/ast/keyword.rs b/src/lib/syntax/ast/keyword.rs index 1307471a71..8f12793a16 100644 --- a/src/lib/syntax/ast/keyword.rs +++ b/src/lib/syntax/ast/keyword.rs @@ -8,6 +8,8 @@ use std::str::FromStr; /// A Javascript Keyword /// As specificed by https://www.ecma-international.org/ecma-262/#sec-keywords pub enum Keyword { + /// The `await` keyword + Await, /// The `break` keyword Break, /// The `case` keyword @@ -32,6 +34,8 @@ pub enum Keyword { Else, /// The `enum` keyword Enum, + /// The `export` keyword + Export, /// The `extends` keyword Extends, /// The `finally` keyword @@ -74,6 +78,8 @@ pub enum Keyword { While, /// The `with` keyword With, + /// The 'yield' keyword + Yield, } #[derive(Debug, Clone)] @@ -99,6 +105,7 @@ impl FromStr for Keyword { type Err = KeywordError; fn from_str(s: &str) -> Result { match s { + "await" => Ok(Await), "break" => Ok(Break), "case" => Ok(Case), "catch" => Ok(Catch), @@ -112,6 +119,7 @@ impl FromStr for Keyword { "else" => Ok(Else), "enum" => Ok(Enum), "extends" => Ok(Extends), + "export" => Ok(Export), "finally" => Ok(Finally), "for" => Ok(For), "function" => Ok(Function), @@ -132,6 +140,7 @@ impl FromStr for Keyword { "void" => Ok(Void), "while" => Ok(While), "with" => Ok(With), + "yield" => Ok(Yield), _ => Err(KeywordError), } } @@ -142,6 +151,7 @@ impl Display for Keyword { f, "{}", match *self { + Await => "await", Break => "break", Case => "case", Catch => "catch", @@ -155,6 +165,7 @@ impl Display for Keyword { Else => "else", Enum => "enum", Extends => "extends", + Export => "export", Finally => "finally", For => "for", Function => "function", @@ -175,6 +186,7 @@ impl Display for Keyword { Void => "void", While => "while", With => "with", + Yield => "yield", } ) } diff --git a/src/lib/syntax/ast/punc.rs b/src/lib/syntax/ast/punc.rs index 0210380dbd..3efdc94af5 100644 --- a/src/lib/syntax/ast/punc.rs +++ b/src/lib/syntax/ast/punc.rs @@ -2,104 +2,106 @@ use std::fmt::{Display, Error, Formatter}; #[derive(PartialEq, Clone, Debug)] /// Punctuation pub enum Punctuator { - /// `{` - OpenBlock, + /// `+` + Add, + /// `&` + And, + /// `=>` + Arrow, + /// `=` + Assign, + /// `+=` + AssignAdd, + /// `&=` + AssignAnd, + /// `/=` + AssignDiv, + /// `<<=` + AssignLeftSh, + /// `%=` + AssignMod, + /// `*=` + AssignMul, + /// `|=` + AssignOr, + /// `>>>=` + AssignRightSh, + /// `-=` + AssignSub, + /// `>>>=` + AssignURightSh, + /// `^=` + AssignXor, + /// `&&` + BoolAnd, + /// `||` + BoolOr, /// `}` CloseBlock, - /// `(` - OpenParen, - /// `)` - CloseParen, - /// `[` - OpenBracket, /// `]` CloseBracket, - /// `.` - Dot, - /// `;` - Semicolon, + /// `)` + CloseParen, + /// `:` + Colon, /// `,` Comma, - /// `<` - LessThan, + /// `--` + Dec, + /// `/` + Div, + /// `.` + Dot, + /// `==` + Eq, /// `>` GreaterThan, - /// `<=` - LessThanOrEq, /// `>=` GreaterThanOrEq, - /// `==` - Eq, + /// `++` + Inc, + /// `<<` + LeftSh, + /// `<` + LessThan, + /// `<=` + LessThanOrEq, + /// `%` + Mod, + /// `*` + Mul, + /// `~` + Neg, + /// `!` + Not, /// `!=` NotEq, + /// `{` + OpenBlock, + /// `[` + OpenBracket, + /// `(` + OpenParen, + /// `|` + Or, + /// `?` + Question, + /// `>>` + RightSh, + /// `;` + Semicolon, + /// `...` + Spread, /// `===` StrictEq, /// `!==` StrictNotEq, - /// `+` - Add, /// `-` Sub, - /// `*` - Mul, - /// `/` - Div, - /// `%` - Mod, - /// `++` - Inc, - /// `--` - Dec, - /// `<<` - LeftSh, - /// `>>` - RightSh, /// `>>>` URightSh, - /// `&` - And, - /// `|` - Or, /// `^` Xor, - /// `!` - Not, - /// `~` - Neg, - /// `&&` - BoolAnd, - /// `||` - BoolOr, - /// `?` - Question, - /// `:` - Colon, - /// `=` - Assign, - /// `+=` - AssignAdd, - /// `-=` - AssignSub, - /// `*=` - AssignMul, - /// `/=` - AssignDiv, - /// `%=` - AssignMod, - /// `<<=` - AssignLeftSh, - /// `>>=` - AssignRightSh, - /// `>>>=` - AssignURightSh, - /// `&=` - AssignAnd, - /// `|=` - AssignOr, - /// `^=` - AssignXor, - /// `=>` - Arrow, } impl Display for Punctuator { fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { @@ -107,55 +109,56 @@ impl Display for Punctuator { f, "{}", match self { - Punctuator::OpenBlock => "{", + Punctuator::Add => "+", + Punctuator::And => "&", + Punctuator::Arrow => "=>", + Punctuator::Assign => "=", + Punctuator::AssignAdd => "+=", + Punctuator::AssignAnd => "&=", + Punctuator::AssignDiv => "/=", + Punctuator::AssignLeftSh => "<<=", + Punctuator::AssignMod => "%=", + Punctuator::AssignMul => "*=", + Punctuator::AssignOr => "|=", + Punctuator::AssignRightSh => ">>=", + Punctuator::AssignSub => "-=", + Punctuator::AssignURightSh => ">>>=", + Punctuator::AssignXor => "^=", + Punctuator::BoolAnd => "&&", + Punctuator::BoolOr => "||", Punctuator::CloseBlock => "}", - Punctuator::OpenParen => "(", - Punctuator::CloseParen => ")", - Punctuator::OpenBracket => "[", Punctuator::CloseBracket => "]", - Punctuator::Dot => ".", - Punctuator::Semicolon => ";", + Punctuator::CloseParen => ")", + Punctuator::Colon => ":", Punctuator::Comma => ",", - Punctuator::LessThan => "<", + Punctuator::Dec => "--", + Punctuator::Div => "/", + Punctuator::Dot => ".", + Punctuator::Eq => "==", Punctuator::GreaterThan => ">", - Punctuator::LessThanOrEq => "<=", Punctuator::GreaterThanOrEq => ">=", - Punctuator::Eq => "==", + Punctuator::Inc => "++", + Punctuator::LeftSh => "<<", + Punctuator::LessThan => "<", + Punctuator::LessThanOrEq => "<=", + Punctuator::Mod => "%", + Punctuator::Mul => "*", + Punctuator::Neg => "~", + Punctuator::Not => "!", Punctuator::NotEq => "!=", + Punctuator::OpenBlock => "{", + Punctuator::OpenBracket => "[", + Punctuator::OpenParen => "(", + Punctuator::Or => "|", + Punctuator::Question => "?", + Punctuator::RightSh => ">>", + Punctuator::Semicolon => ";", + Punctuator::Spread => "...", Punctuator::StrictEq => "===", Punctuator::StrictNotEq => "!==", - Punctuator::Add => "+", Punctuator::Sub => "-", - Punctuator::Mul => "*", - Punctuator::Div => "/", - Punctuator::Mod => "%", - Punctuator::Inc => "++", - Punctuator::Dec => "--", - Punctuator::LeftSh => "<<", - Punctuator::RightSh => ">>", Punctuator::URightSh => ">>>", - Punctuator::And => "&", - Punctuator::Or => "|", Punctuator::Xor => "^", - Punctuator::Not => "!", - Punctuator::Neg => "~", - Punctuator::BoolAnd => "&&", - Punctuator::BoolOr => "||", - Punctuator::Question => "?", - Punctuator::Colon => ":", - Punctuator::Assign => "=", - Punctuator::AssignAdd => "+=", - Punctuator::AssignSub => "-=", - Punctuator::AssignMul => "*=", - Punctuator::AssignDiv => "/=", - Punctuator::AssignMod => "%=", - Punctuator::AssignLeftSh => "<<=", - Punctuator::AssignRightSh => ">>=", - Punctuator::AssignURightSh => ">>>=", - Punctuator::AssignAnd => "&=", - Punctuator::AssignOr => "|=", - Punctuator::AssignXor => "^=", - Punctuator::Arrow => "=>", } ) } diff --git a/src/lib/syntax/lexer.rs b/src/lib/syntax/lexer.rs index c9510bc13f..417a5b6fa4 100644 --- a/src/lib/syntax/lexer.rs +++ b/src/lib/syntax/lexer.rs @@ -436,7 +436,18 @@ impl<'a> Lexer<'a> { } ';' => self.push_punc(Punctuator::Semicolon), ':' => self.push_punc(Punctuator::Colon), - '.' => self.push_punc(Punctuator::Dot), + '.' => { + // . or ... + if self.next_is('.') { + if self.next_is('.') { + self.push_punc(Punctuator::Spread); + } else { + return Err(LexerError::new("Expecting Token .")); + } + } else { + self.push_punc(Punctuator::Dot); + }; + } '(' => self.push_punc(Punctuator::OpenParen), ')' => self.push_punc(Punctuator::CloseParen), ',' => self.push_punc(Punctuator::Comma), @@ -543,11 +554,172 @@ mod tests { use super::*; use crate::syntax::ast::keyword::Keyword; + #[test] + fn check_string() { + let s = "'aaa' \"bbb\""; + let mut lexer = Lexer::new(s); + lexer.lex().unwrap(); + assert_eq!( + lexer.tokens[0].data, + TokenData::StringLiteral("aaa".to_string()) + ); + + assert_eq!( + lexer.tokens[1].data, + TokenData::StringLiteral("bbb".to_string()) + ); + } + + #[test] + fn check_punctuators() { + // TODO: Support ** ++ + // https://tc39.es/ecma262/#sec-punctuators + let s = "{ ( ) [ ] . ... ; , < > <= >= == != === !== \ + + - * % -- << >> >>> & | ^ ! ~ && || ? : \ + = += -= *= &= **= <<= >>= >>>= &= |= ^= =>"; + let mut lexer = Lexer::new(s); + lexer.lex().unwrap(); + assert_eq!( + lexer.tokens[0].data, + TokenData::Punctuator(Punctuator::OpenBlock) + ); + assert_eq!( + lexer.tokens[1].data, + TokenData::Punctuator(Punctuator::OpenParen) + ); + assert_eq!( + lexer.tokens[2].data, + TokenData::Punctuator(Punctuator::CloseParen) + ); + assert_eq!( + lexer.tokens[3].data, + TokenData::Punctuator(Punctuator::OpenBracket) + ); + assert_eq!( + lexer.tokens[4].data, + TokenData::Punctuator(Punctuator::CloseBracket) + ); + assert_eq!(lexer.tokens[5].data, TokenData::Punctuator(Punctuator::Dot)); + assert_eq!( + lexer.tokens[6].data, + TokenData::Punctuator(Punctuator::Spread) + ); + assert_eq!( + lexer.tokens[7].data, + TokenData::Punctuator(Punctuator::Semicolon) + ); + assert_eq!( + lexer.tokens[8].data, + TokenData::Punctuator(Punctuator::Comma) + ); + assert_eq!( + lexer.tokens[9].data, + TokenData::Punctuator(Punctuator::LessThan) + ); + assert_eq!( + lexer.tokens[10].data, + TokenData::Punctuator(Punctuator::GreaterThan) + ); + assert_eq!( + lexer.tokens[11].data, + TokenData::Punctuator(Punctuator::LessThanOrEq) + ); + assert_eq!( + lexer.tokens[12].data, + TokenData::Punctuator(Punctuator::GreaterThanOrEq) + ); + assert_eq!(lexer.tokens[13].data, TokenData::Punctuator(Punctuator::Eq)); + assert_eq!( + lexer.tokens[14].data, + TokenData::Punctuator(Punctuator::NotEq) + ); + assert_eq!( + lexer.tokens[15].data, + TokenData::Punctuator(Punctuator::StrictEq) + ); + assert_eq!( + lexer.tokens[16].data, + TokenData::Punctuator(Punctuator::StrictNotEq) + ); + assert_eq!( + lexer.tokens[17].data, + TokenData::Punctuator(Punctuator::Add) + ); + assert_eq!( + lexer.tokens[18].data, + TokenData::Punctuator(Punctuator::Sub) + ); + assert_eq!( + lexer.tokens[19].data, + TokenData::Punctuator(Punctuator::Mul) + ); + assert_eq!( + lexer.tokens[20].data, + TokenData::Punctuator(Punctuator::Mod) + ); + assert_eq!( + lexer.tokens[21].data, + TokenData::Punctuator(Punctuator::Dec) + ); + assert_eq!( + lexer.tokens[22].data, + TokenData::Punctuator(Punctuator::LeftSh) + ); + } + + #[test] + fn check_keywords() { + // https://tc39.es/ecma262/#sec-keywords + let s = "await break case catch class const continue debugger default delete \ + do else export extends finally for function if import in instanceof \ + new return super switch this throw try typeof var void while with yield"; + + let mut lexer = Lexer::new(s); + lexer.lex().unwrap(); + assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Await)); + assert_eq!(lexer.tokens[1].data, TokenData::Keyword(Keyword::Break)); + assert_eq!(lexer.tokens[2].data, TokenData::Keyword(Keyword::Case)); + assert_eq!(lexer.tokens[3].data, TokenData::Keyword(Keyword::Catch)); + assert_eq!(lexer.tokens[4].data, TokenData::Keyword(Keyword::Class)); + assert_eq!(lexer.tokens[5].data, TokenData::Keyword(Keyword::Const)); + assert_eq!(lexer.tokens[6].data, TokenData::Keyword(Keyword::Continue)); + assert_eq!(lexer.tokens[7].data, TokenData::Keyword(Keyword::Debugger)); + assert_eq!(lexer.tokens[8].data, TokenData::Keyword(Keyword::Default)); + assert_eq!(lexer.tokens[9].data, TokenData::Keyword(Keyword::Delete)); + assert_eq!(lexer.tokens[10].data, TokenData::Keyword(Keyword::Do)); + assert_eq!(lexer.tokens[11].data, TokenData::Keyword(Keyword::Else)); + assert_eq!(lexer.tokens[12].data, TokenData::Keyword(Keyword::Export)); + assert_eq!(lexer.tokens[13].data, TokenData::Keyword(Keyword::Extends)); + assert_eq!(lexer.tokens[14].data, TokenData::Keyword(Keyword::Finally)); + assert_eq!(lexer.tokens[15].data, TokenData::Keyword(Keyword::For)); + assert_eq!(lexer.tokens[16].data, TokenData::Keyword(Keyword::Function)); + assert_eq!(lexer.tokens[17].data, TokenData::Keyword(Keyword::If)); + assert_eq!(lexer.tokens[18].data, TokenData::Keyword(Keyword::Import)); + assert_eq!(lexer.tokens[19].data, TokenData::Keyword(Keyword::In)); + assert_eq!( + lexer.tokens[20].data, + TokenData::Keyword(Keyword::InstanceOf) + ); + assert_eq!(lexer.tokens[21].data, TokenData::Keyword(Keyword::New)); + assert_eq!(lexer.tokens[22].data, TokenData::Keyword(Keyword::Return)); + assert_eq!(lexer.tokens[23].data, TokenData::Keyword(Keyword::Super)); + assert_eq!(lexer.tokens[24].data, TokenData::Keyword(Keyword::Switch)); + assert_eq!(lexer.tokens[25].data, TokenData::Keyword(Keyword::This)); + assert_eq!(lexer.tokens[26].data, TokenData::Keyword(Keyword::Throw)); + assert_eq!(lexer.tokens[27].data, TokenData::Keyword(Keyword::Try)); + assert_eq!(lexer.tokens[28].data, TokenData::Keyword(Keyword::TypeOf)); + assert_eq!(lexer.tokens[29].data, TokenData::Keyword(Keyword::Var)); + assert_eq!(lexer.tokens[30].data, TokenData::Keyword(Keyword::Void)); + assert_eq!(lexer.tokens[31].data, TokenData::Keyword(Keyword::While)); + assert_eq!(lexer.tokens[32].data, TokenData::Keyword(Keyword::With)); + assert_eq!(lexer.tokens[33].data, TokenData::Keyword(Keyword::Yield)); + } + #[test] fn check_variable_definition_tokens() { let s = &String::from("let a = 'hello';"); let mut lexer = Lexer::new(s); - lexer.lex().expect("finished"); + lexer.lex().unwrap(); assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Let)); assert_eq!(lexer.tokens[1].data, TokenData::Identifier("a".to_string())); assert_eq!( @@ -565,7 +737,7 @@ mod tests { let s = &String::from("console.log(\"hello world\");"); // -------------------123456789 let mut lexer = Lexer::new(s); - lexer.lex().expect("finished"); + lexer.lex().unwrap(); // The first column is 1 (not zero indexed) assert_eq!(lexer.tokens[0].pos.column_number, 1); assert_eq!(lexer.tokens[0].pos.line_number, 1); @@ -595,7 +767,7 @@ mod tests { // Here we want an example of decrementing an integer let s = &String::from("let a = b--;"); let mut lexer = Lexer::new(s); - lexer.lex().expect("finished"); + lexer.lex().unwrap(); assert_eq!(lexer.tokens[4].data, TokenData::Punctuator(Punctuator::Dec)); // Decrementing means adding 2 characters '--', the lexer should consume it as a single token // and move the curser forward by 2, meaning the next token should be a semicolon @@ -608,7 +780,7 @@ mod tests { #[test] fn numbers() { let mut lexer = Lexer::new("1 2 0x34 056 7.89 5e3 5e+3 5e-3 0b10 0O123 0999"); - lexer.lex().expect("finished"); + lexer.lex().unwrap(); assert_eq!(lexer.tokens[0].data, TokenData::NumericLiteral(1.0)); assert_eq!(lexer.tokens[1].data, TokenData::NumericLiteral(2.0)); assert_eq!(lexer.tokens[2].data, TokenData::NumericLiteral(52.0)); @@ -625,7 +797,7 @@ mod tests { #[test] fn test_single_number_without_semicolon() { let mut lexer = Lexer::new("1"); - lexer.lex().expect("finished"); + lexer.lex().unwrap(); dbg!(lexer.tokens); }