From 99bdea009cdfdb543353e7c85a1b63a3a594399b Mon Sep 17 00:00:00 2001 From: Jason Williams Date: Tue, 14 Aug 2018 20:00:48 +0100 Subject: [PATCH] more code added, but with errors --- src/lib/syntax/ast/punc.rs | 196 ++++++++++++++++++------------------- src/lib/syntax/lexer.rs | 114 +++++++++++++++++++-- 2 files changed, 204 insertions(+), 106 deletions(-) diff --git a/src/lib/syntax/ast/punc.rs b/src/lib/syntax/ast/punc.rs index c7ca41ac0b..d48d6bdb51 100644 --- a/src/lib/syntax/ast/punc.rs +++ b/src/lib/syntax/ast/punc.rs @@ -3,103 +3,103 @@ use std::fmt::{Display, Error, Formatter}; /// Punctuation pub enum Punctuator { /// `{` - POpenBlock, + OpenBlock, /// `}` - PCloseBlock, + CloseBlock, /// `(` - POpenParen, + OpenParen, /// `)` - PCloseParen, + CloseParen, /// `[` - POpenBracket, + OpenBracket, /// `]` - PCloseBracket, + CloseBracket, /// `.` - PDot, + Dot, /// `;` - PSemicolon, + Semicolon, /// `,` - PComma, + Comma, /// `<` - PLessThan, + LessThan, /// `>` - PGreaterThan, + GreaterThan, /// `<=` - PLessThanOrEq, + LessThanOrEq, /// `>=` - PGreaterThanOrEq, + GreaterThanOrEq, /// `==` - PEq, + Eq, /// `!=` - PNotEq, + NotEq, /// `===` - PStrictEq, + StrictEq, /// `!==` - PStrictNotEq, + StrictNotEq, /// `+` - PAdd, + Add, /// `-` - PSub, + Sub, /// `*` - PMul, + Mul, /// `/` - PDiv, + Div, /// `%` - PMod, + Mod, /// `++` - PInc, + Inc, /// `--` - PDec, + Dec, /// `<<` - PLeftSh, + LeftSh, /// `>>` - PRightSh, + RightSh, /// `>>>` - PURightSh, + URightSh, /// `&` - PAnd, + And, /// `|` - POr, + Or, /// `^` - PXor, + Xor, /// `!` - PNot, + Not, /// `~` - PNeg, + Neg, /// `&&` - PBoolAnd, + BoolAnd, /// `||` - PBoolOr, + BoolOr, /// `?` - PQuestion, + Question, /// `:` - PColon, + Colon, /// `=` - PAssign, + Assign, /// `+=` - PAssignAdd, + AssignAdd, /// `-=` - PAssignSub, + AssignSub, /// `*=` - PAssignMul, + AssignMul, /// `/=` - PAssignDiv, + AssignDiv, /// `%=` - PAssignMod, + AssignMod, /// `<<=` - PAssignLeftSh, + AssignLeftSh, /// `>>=` - PAssignRightSh, + AssignRightSh, /// `>>>=` - PAssignURightSh, + AssignURightSh, /// `&=` - PAssignAnd, + AssignAnd, /// `|=` - PAssignOr, + AssignOr, /// `^=` - PAssignXor, + AssignXor, /// `=>` - PArrow, + Arrow, } impl Display for Punctuator { fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { @@ -107,55 +107,55 @@ impl Display for Punctuator { f, "{}", match self { - Punctuator::POpenBlock => "{", - Punctuator::PCloseBlock => "}", - Punctuator::POpenParen => "(", - Punctuator::PCloseParen => ")", - Punctuator::POpenBracket => "[", - Punctuator::PCloseBracket => "]", - Punctuator::PDot => ".", - Punctuator::PSemicolon => ";", - Punctuator::PComma => ",", - Punctuator::PLessThan => "<", - Punctuator::PGreaterThan => ">", - Punctuator::PLessThanOrEq => "<=", - Punctuator::PGreaterThanOrEq => ">=", - Punctuator::PEq => "==", - Punctuator::PNotEq => "!=", - Punctuator::PStrictEq => "===", - Punctuator::PStrictNotEq => "!==", - Punctuator::PAdd => "+", - Punctuator::PSub => "-", - Punctuator::PMul => "*", - Punctuator::PDiv => "/", - Punctuator::PMod => "%", - Punctuator::PInc => "++", - Punctuator::PDec => "--", - Punctuator::PLeftSh => "<<", - Punctuator::PRightSh => ">>", - Punctuator::PURightSh => ">>>", - Punctuator::PAnd => "&", - Punctuator::POr => "|", - Punctuator::PXor => "^", - Punctuator::PNot => "!", - Punctuator::PNeg => "~", - Punctuator::PBoolAnd => "&&", - Punctuator::PBoolOr => "||", - Punctuator::PQuestion => "?", - Punctuator::PColon => ":", - Punctuator::PAssign => "=", - Punctuator::PAssignAdd => "+=", - Punctuator::PAssignSub => "-=", - Punctuator::PAssignMul => "*=", - Punctuator::PAssignDiv => "/=", - Punctuator::PAssignMod => "%=", - Punctuator::PAssignLeftSh => "<<=", - Punctuator::PAssignRightSh => ">>=", - Punctuator::PAssignURightSh => ">>>=", - Punctuator::PAssignAnd => "&=", - Punctuator::PAssignOr => "|=", - Punctuator::PAssignXor => "^=", - Punctuator::PArrow => "=>", + Punctuator::OpenBlock => "{", + Punctuator::CloseBlock => "}", + Punctuator::OpenParen => "(", + Punctuator::CloseParen => ")", + Punctuator::OpenBracket => "[", + Punctuator::CloseBracket => "]", + Punctuator::Dot => ".", + Punctuator::Semicolon => ";", + Punctuator::Comma => ",", + Punctuator::LessThan => "<", + Punctuator::GreaterThan => ">", + Punctuator::LessThanOrEq => "<=", + Punctuator::GreaterThanOrEq => ">=", + Punctuator::Eq => "==", + Punctuator::NotEq => "!=", + Punctuator::StrictEq => "===", + Punctuator::StrictNotEq => "!==", + Punctuator::Add => "+", + Punctuator::Sub => "-", + Punctuator::Mul => "*", + Punctuator::Div => "/", + Punctuator::Mod => "%", + Punctuator::Inc => "++", + Punctuator::Dec => "--", + Punctuator::LeftSh => "<<", + Punctuator::RightSh => ">>", + Punctuator::URightSh => ">>>", + Punctuator::And => "&", + Punctuator::Or => "|", + Punctuator::Xor => "^", + Punctuator::Not => "!", + Punctuator::Neg => "~", + Punctuator::BoolAnd => "&&", + Punctuator::BoolOr => "||", + Punctuator::Question => "?", + Punctuator::Colon => ":", + Punctuator::Assign => "=", + Punctuator::AssignAdd => "+=", + Punctuator::AssignSub => "-=", + Punctuator::AssignMul => "*=", + Punctuator::AssignDiv => "/=", + Punctuator::AssignMod => "%=", + Punctuator::AssignLeftSh => "<<=", + Punctuator::AssignRightSh => ">>=", + Punctuator::AssignURightSh => ">>>=", + Punctuator::AssignAnd => "&=", + Punctuator::AssignOr => "|=", + Punctuator::AssignXor => "^=", + Punctuator::Arrow => "=>", } ) } diff --git a/src/lib/syntax/lexer.rs b/src/lib/syntax/lexer.rs index dc11a50d10..d1d67c5921 100644 --- a/src/lib/syntax/lexer.rs +++ b/src/lib/syntax/lexer.rs @@ -1,9 +1,9 @@ -use serialize::json::from_str; use std::char::from_u32; use std::error; use std::fmt; use std::iter::Peekable; use std::str::Chars; +use std::str::FromStr; use syntax::ast::punc::Punctuator; use syntax::ast::token::{Token, TokenData}; @@ -75,6 +75,24 @@ impl<'a> Lexer<'a> { self.buffer.next().ok_or(LexerError::new("next failed")) } + /// Attempt to read until the end of the line and return the string + fn read_line(&mut self) -> Result { + let mut buf = String::new(); + loop { + let ch = self.next()?; + match ch { + _ if ch.is_ascii_control() => { + break; + } + _ => { + buf.push(ch); + } + } + } + + Ok(buf) + } + fn preview_next(&mut self) -> Result<&char, LexerError> { // ok_or converts Option to a Result self.buffer @@ -175,8 +193,7 @@ impl<'a> Lexer<'a> { let ch = self.preview_next()?; match ch { ch if ch.is_digit(16) => { - self.next()?; - buf.push(*ch); + buf.push(self.next()?); } _ => break, } @@ -199,14 +216,95 @@ impl<'a> Lexer<'a> { _ => break, } } - if gone_decimal { - from_str(&buf) - } else { - u64::from_str_radix(&buf, 8) + u64::from_str_radix(&buf, 8).unwrap() + }; + self.push_token(TokenData::TNumericLiteral(num as f64)) + } + _ if ch.is_digit(10) => { + let mut buf = ch.to_string(); + loop { + let ch = self.preview_next()?; + match ch { + '.' => { + buf.push(self.next()?); + } + _ if ch.is_digit(10) => { + buf.push(self.next()?); + } + _ => break, + } + } + // TODO make this a bit more safe -------------------------------VVVV + self.push_token(TokenData::TNumericLiteral(f64::from_str(&buf).unwrap())) + } + _ if ch.is_alphabetic() || ch == '$' || ch == '_' => { + let mut buf = ch.to_string(); + loop { + let ch = self.preview_next()?; + match ch { + _ if ch.is_alphabetic() || ch.is_digit(10) || *ch == '_' => { + buf.push(self.next()?); + } + _ => { + break; + } + } + } + // Match won't compare &String to &str so i need to convert it :( + let buf_compare: &str = &buf; + self.push_token(match buf_compare { + "true" => TokenData::TBooleanLiteral(true), + "false" => TokenData::TBooleanLiteral(false), + "null" => TokenData::TNullLiteral, + slice => match FromStr::from_str(slice) { + Ok(keyword) => TokenData::TKeyword(keyword), + Err(e) => TokenData::TIdentifier(buf.clone()), + }, + }); + } + ';' => self.push_punc(Punctuator::Semicolon), + ':' => self.push_punc(Punctuator::Colon), + '.' => self.push_punc(Punctuator::Dot), + '(' => self.push_punc(Punctuator::OpenParen), + ')' => self.push_punc(Punctuator::CloseParen), + ',' => self.push_punc(Punctuator::Comma), + '{' => self.push_punc(Punctuator::OpenBlock), + '}' => self.push_punc(Punctuator::CloseBlock), + '[' => self.push_punc(Punctuator::OpenBracket), + ']' => self.push_punc(Punctuator::CloseBracket), + '?' => self.push_punc(Punctuator::Question), + '/' => { + let token = match self.preview_next()? { + // Matched comment + '/' => { + let comment = self.read_line()?; + TokenData::TComment(comment) + } + '*' => { + let mut buf = String::new(); + loop { + match self.next()? { + '*' => { + if self.next_is('/')? { + break; + } else { + buf.push('*') + } + } + ch => buf.push(ch), + } + } + TokenData::TComment(buf) } + '=' => TokenData::TPunctuator(Punctuator::AssignDiv), + _ => TokenData::TPunctuator(Punctuator::Div), }; - self.push_token(TokenData::TNumericLiteral(num)) + self.push_token(token) } + ch => panic!( + "{}:{}: Unexpected '{}'", + self.line_number, self.column_number, ch + ), } } }