//! Some environments are stored as JSObjects. This is for GC, i.e we want to keep an environment if a variable is closed-over (a closure is returned).
//! Some environments are stored as `JSObjects`. This is for GC, i.e we want to keep an environment if a variable is closed-over (a closure is returned).
//! All of the logic to handle scope/environment records are stored in here.
//! All of the logic to handle scope/environment records are stored in here.
//!
//!
//! There are 5 Environment record kinds. They all have methods in common, these are implemented as a the `EnvironmentRecordTrait`
//! There are 5 Environment record kinds. They all have methods in common, these are implemented as a the `EnvironmentRecordTrait`
/// Returns a single element String containing the code unit at index pos within the String value resulting from converting this object to a String. If there is no element at that index, the result is the empty String. The result is a String value, not a String object.
/// Returns a single element String containing the code unit at index pos within the String value
/// Returns a Number (a nonnegative integer less than 216) that is the numeric value of the code unit at index pos within the String resulting from converting this object to a String. If there is no element at that index, the result is NaN.
/// Returns a Number (a nonnegative integer less than 216) that is the numeric value of the code
//! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure.
//! The Lexer splits its input source code into a sequence of input elements called tokens, represented by the [Token](../ast/token/struct.Token.html) structure.
//! It also removes whitespace and comments and attaches them to the next token.
//! It also removes whitespace and comments and attaches them to the next token.
usecrate::syntax::ast::punc::Punctuator;
usecrate::syntax::ast::{
usecrate::syntax::ast::token::{Token,TokenData};
punc::Punctuator,
usestd::char::{decode_utf16,from_u32};
token::{Token,TokenData},
usestd::error;
};
usestd::fmt;
usestd::iter::Peekable;
usestd::{
usestd::str::Chars;
char::{decode_utf16,from_u32},
usestd::str::FromStr;
error,fmt,
iter::Peekable,
#[allow(unused)]
str::{Chars,FromStr},
};
macro_rules!vop{
macro_rules!vop{
($this:ident,$assign_op:expr,$op:expr)=>({
($this:ident,$assign_op:expr,$op:expr)=>({
letpreview=$this.preview_next().unwrap();
letpreview=$this.preview_next().unwrap();
@ -71,8 +73,8 @@ pub struct LexerError {
}
}
implLexerError{
implLexerError{
fnnew(msg: &str)-> LexerError{
fnnew(msg: &str)-> Self{
LexerError{
Self{
details: msg.to_string(),
details: msg.to_string(),
}
}
}
}
@ -96,6 +98,7 @@ impl error::Error for LexerError {
}
}
/// A lexical analyzer for JavaScript source code
/// A lexical analyzer for JavaScript source code
#[derive(Debug)]
pubstructLexer<'a>{
pubstructLexer<'a>{
// The list fo tokens generated so far
// The list fo tokens generated so far
pubtokens: Vec<Token>,
pubtokens: Vec<Token>,
@ -143,7 +146,7 @@ impl<'a> Lexer<'a> {
/// next fetches the next token and return it, or a LexerError if there are no more.
/// next fetches the next token and return it, or a LexerError if there are no more.
fnnext(&mutself)-> Result<char,LexerError>{
fnnext(&mutself)-> Result<char,LexerError>{
matchself.buffer.next(){
matchself.buffer.next(){
Some(char)=>Ok(char),
Some(ch)=>Ok(ch),
None=>Err(LexerError::new("finished")),
None=>Err(LexerError::new("finished")),
}
}
}
}
@ -168,11 +171,7 @@ impl<'a> Lexer<'a> {
/// Preview the next character but don't actually increment
/// Preview the next character but don't actually increment
fnpreview_next(&mutself)-> Option<char>{
fnpreview_next(&mutself)-> Option<char>{
// No need to return a reference, we can return a copy
self.buffer.peek().copied()
matchself.buffer.peek(){
Some(v)=>Some(*v),
None=>None,
}
}
}
/// Utility Function, while ``f(char)`` is true, read chars and move curser.
/// Utility Function, while ``f(char)`` is true, read chars and move curser.
@ -191,10 +190,7 @@ impl<'a> Lexer<'a> {
/// next_is compares the character passed in to the next character, if they match true is returned and the buffer is incremented
/// next_is compares the character passed in to the next character, if they match true is returned and the buffer is incremented
fnnext_is(&mutself,peek: char)-> bool{
fnnext_is(&mutself,peek: char)-> bool{
letresult=matchself.preview_next(){
letresult=self.preview_next()==Some(peek);
Some(v)=>v==peek,
None=>false,
};
ifresult{
ifresult{
self.buffer.next();
self.buffer.next();
}
}
@ -204,11 +200,8 @@ impl<'a> Lexer<'a> {
pubfnlex(&mutself)-> Result<(),LexerError>{
pubfnlex(&mutself)-> Result<(),LexerError>{
loop{
loop{
// Check if we've reached the end
// Check if we've reached the end
matchself.preview_next(){
ifself.preview_next().is_none(){
Some(_)=>(),// If there are still characters, carry on
returnOk(());
None=>{
returnOk(());
}
}
}
self.column_number+=1;
self.column_number+=1;
letch=self.next()?;
letch=self.next()?;
@ -235,7 +228,7 @@ impl<'a> Lexer<'a> {
'0'=>'\0',
'0'=>'\0',
'x'=>{
'x'=>{
letmutnums=String::with_capacity(2);
letmutnums=String::with_capacity(2);
for_in0u8..2{
for_in0_u8..2{
nums.push(self.next()?);
nums.push(self.next()?);
}
}
self.column_number+=2;
self.column_number+=2;
@ -260,7 +253,7 @@ impl<'a> Lexer<'a> {
// Support \u{X..X} (Unicode Codepoint)
// Support \u{X..X} (Unicode Codepoint)
ifself.next_is('{'){
ifself.next_is('{'){
lets=self
lets=self
.take_char_while(|c|c.is_alphanumeric())
.take_char_while(char::is_alphanumeric)
.unwrap();
.unwrap();
// We know this is a single unicode codepoint, convert to u32
// We know this is a single unicode codepoint, convert to u32
@ -279,7 +272,7 @@ impl<'a> Lexer<'a> {
loop{
loop{
// Collect each character after \u e.g \uD83D will give "D83D"
// Collect each character after \u e.g \uD83D will give "D83D"
lets=self
lets=self
.take_char_while(|c|c.is_alphanumeric())
.take_char_while(char::is_alphanumeric)
.unwrap();
.unwrap();
// Convert to u16
// Convert to u16
@ -300,12 +293,10 @@ impl<'a> Lexer<'a> {
// codepoints length should either be 1 (unicode codepoint) or 2 (surrogate codepoint).
// codepoints length should either be 1 (unicode codepoint) or 2 (surrogate codepoint).
// Rust's decode_utf16 will deal with it regardless
// Rust's decode_utf16 will deal with it regardless
Iban Eguia
5 years ago
committed by
Jason Williams