github
/
boa
mirror of https://github.com/boa-dev/boa.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

removing T from token

pull/1/head
jasonwilliams 6 years ago
parent
bd9b60a53b
commit
686a7368e3
2 changed files with 33 additions and 33 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 40
      src/lib/syntax/ast/token.rs
  2. 26
      src/lib/syntax/lexer.rs

40
src/lib/syntax/ast/token.rs
Unescape View File

@ -25,40 +25,40 @@ impl Token {
/// Represents the type of Token /// Represents the type of Token
pub enum TokenData { pub enum TokenData {
/// A boolean literal, which is either `true` or `false` /// A boolean literal, which is either `true` or `false`
TBooleanLiteral(bool), BooleanLiteral(bool),
/// The end of the file /// The end of the file
TEOF, EOF,
/// An identifier /// An identifier
TIdentifier(String), Identifier(String),
/// A keyword /// A keyword
TKeyword(Keyword), Keyword(Keyword),
/// A `null` literal /// A `null` literal
TNullLiteral, NullLiteral,
/// A numeric literal /// A numeric literal
TNumericLiteral(f64), NumericLiteral(f64),
/// A piece of punctuation /// A piece of punctuation
TPunctuator(Punctuator), Punctuator(Punctuator),
/// A string literal /// A string literal
TStringLiteral(String), StringLiteral(String),
/// A regular expression /// A regular expression
TRegularExpression(String), RegularExpression(String),
/// A comment /// A comment
TComment(String), Comment(String),
} }
impl Display for TokenData { impl Display for TokenData {
fn fmt(&self, f: &mut Formatter) -> Result { fn fmt(&self, f: &mut Formatter) -> Result {
match self.clone() { match self.clone() {
TokenData::TBooleanLiteral(val) => write!(f, "{}", val), TokenData::BooleanLiteral(val) => write!(f, "{}", val),
TokenData::TEOF => write!(f, "end of file"), TokenData::EOF => write!(f, "end of file"),
TokenData::TIdentifier(ident) => write!(f, "{}", ident), TokenData::Identifier(ident) => write!(f, "{}", ident),
TokenData::TKeyword(word) => write!(f, "{}", word), TokenData::Keyword(word) => write!(f, "{}", word),
TokenData::TNullLiteral => write!(f, "null"), TokenData::NullLiteral => write!(f, "null"),
TokenData::TNumericLiteral(num) => write!(f, "{}", num), TokenData::NumericLiteral(num) => write!(f, "{}", num),
TokenData::TPunctuator(punc) => write!(f, "{}", punc), TokenData::Punctuator(punc) => write!(f, "{}", punc),
TokenData::TStringLiteral(lit) => write!(f, "{}", lit), TokenData::StringLiteral(lit) => write!(f, "{}", lit),
TokenData::TRegularExpression(reg) => write!(f, "{}", reg), TokenData::RegularExpression(reg) => write!(f, "{}", reg),
TokenData::TComment(comm) => write!(f, "/*{}*/", comm), TokenData::Comment(comm) => write!(f, "/*{}*/", comm),
} }
} }
} }

26
src/lib/syntax/lexer.rs
Unescape View File

@ -68,7 +68,7 @@ impl<'a> Lexer<'a> {
/// Push a punctuation token /// Push a punctuation token
fn push_punc(&mut self, punc: Punctuator) { fn push_punc(&mut self, punc: Punctuator) {
self.push_token(TokenData::TPunctuator(punc)); self.push_token(TokenData::Punctuator(punc));
} }
fn next(&mut self) -> Result<char, LexerError> { fn next(&mut self) -> Result<char, LexerError> {
@ -186,7 +186,7 @@ impl<'a> Lexer<'a> {
ch => buf.push(ch), ch => buf.push(ch),
} }
} }
self.push_token(TokenData::TStringLiteral(buf)); self.push_token(TokenData::StringLiteral(buf));
} }
'0' => { '0' => {
let mut buf = String::new(); let mut buf = String::new();
@ -218,7 +218,7 @@ impl<'a> Lexer<'a> {
} }
u64::from_str_radix(&buf, 8).unwrap() u64::from_str_radix(&buf, 8).unwrap()
}; };
self.push_token(TokenData::TNumericLiteral(num as f64)) self.push_token(TokenData::NumericLiteral(num as f64))
} }
_ if ch.is_digit(10) => { _ if ch.is_digit(10) => {
let mut buf = ch.to_string(); let mut buf = ch.to_string();
@ -235,7 +235,7 @@ impl<'a> Lexer<'a> {
} }
} }
// TODO make this a bit more safe -------------------------------VVVV // TODO make this a bit more safe -------------------------------VVVV
self.push_token(TokenData::TNumericLiteral(f64::from_str(&buf).unwrap())) self.push_token(TokenData::NumericLiteral(f64::from_str(&buf).unwrap()))
} }
_ if ch.is_alphabetic() || ch == '$' || ch == '_' => { _ if ch.is_alphabetic() || ch == '$' || ch == '_' => {
let mut buf = ch.to_string(); let mut buf = ch.to_string();
@ -253,12 +253,12 @@ impl<'a> Lexer<'a> {
// Match won't compare &String to &str so i need to convert it :( // Match won't compare &String to &str so i need to convert it :(
let buf_compare: &str = &buf; let buf_compare: &str = &buf;
self.push_token(match buf_compare { self.push_token(match buf_compare {
"true" => TokenData::TBooleanLiteral(true), "true" => TokenData::BooleanLiteral(true),
"false" => TokenData::TBooleanLiteral(false), "false" => TokenData::BooleanLiteral(false),
"null" => TokenData::TNullLiteral, "null" => TokenData::NullLiteral,
slice => match FromStr::from_str(slice) { slice => match FromStr::from_str(slice) {
Ok(keyword) => TokenData::TKeyword(keyword), Ok(keyword) => TokenData::Keyword(keyword),
Err(_) => TokenData::TIdentifier(buf.clone()), Err(_) => TokenData::Identifier(buf.clone()),
}, },
}); });
} }
@ -278,7 +278,7 @@ impl<'a> Lexer<'a> {
// Matched comment // Matched comment
'/' => { '/' => {
let comment = self.read_line()?; let comment = self.read_line()?;
TokenData::TComment(comment) TokenData::Comment(comment)
} }
'*' => { '*' => {
let mut buf = String::new(); let mut buf = String::new();
@ -294,10 +294,10 @@ impl<'a> Lexer<'a> {
ch => buf.push(ch), ch => buf.push(ch),
} }
} }
TokenData::TComment(buf) TokenData::Comment(buf)
} }
'=' => TokenData::TPunctuator(Punctuator::AssignDiv), '=' => TokenData::Punctuator(Punctuator::AssignDiv),
_ => TokenData::TPunctuator(Punctuator::Div), _ => TokenData::Punctuator(Punctuator::Div),
}; };
self.push_token(token) self.push_token(token)
} }

Write Preview
Loading…
Cancel
Save