From 84b4da545a3472646d7db62c3c56b37bf1c0dc04 Mon Sep 17 00:00:00 2001 From: HalidOdat Date: Wed, 29 Apr 2020 01:37:41 +0200 Subject: [PATCH] Fix #331 "We only get `Const::Num`, never `Const::Int`" (#338) --- boa/src/builtins/array/mod.rs | 4 +- boa/src/builtins/boolean/mod.rs | 2 +- boa/src/builtins/console/tests.rs | 4 +- boa/src/builtins/function/tests.rs | 6 +- boa/src/builtins/number/mod.rs | 2 +- boa/src/builtins/object/mod.rs | 2 +- boa/src/builtins/value/mod.rs | 54 ++- boa/src/exec/mod.rs | 10 +- boa/src/syntax/ast/token.rs | 36 +- boa/src/syntax/lexer/mod.rs | 350 +++++++++++------- boa/src/syntax/lexer/tests.rs | 89 +++-- .../primary/array_initializer/tests.rs | 32 +- .../syntax/parser/expression/primary/mod.rs | 8 +- boa/src/syntax/parser/expression/tests.rs | 18 +- .../parser/statement/declaration/tests.rs | 24 +- .../parser/statement/iteration/tests.rs | 6 +- boa/src/syntax/parser/tests.rs | 2 +- 17 files changed, 403 insertions(+), 246 deletions(-) diff --git a/boa/src/builtins/array/mod.rs b/boa/src/builtins/array/mod.rs index 0aa9a09576..156cd617e6 100644 --- a/boa/src/builtins/array/mod.rs +++ b/boa/src/builtins/array/mod.rs @@ -739,11 +739,11 @@ pub fn find_index(this: &Value, args: &[Value], interpreter: &mut Interpreter) - let result = interpreter.call(predicate_arg, &this_arg, arguments)?; if result.is_true() { - return Ok(Gc::new(ValueData::Number(f64::from(i)))); + return Ok(Gc::new(ValueData::Rational(f64::from(i)))); } } - Ok(Gc::new(ValueData::Number(f64::from(-1)))) + Ok(Gc::new(ValueData::Rational(f64::from(-1)))) } /// `Array.prototype.fill( value[, start[, end]] )` diff --git a/boa/src/builtins/boolean/mod.rs b/boa/src/builtins/boolean/mod.rs index 3813cfa87b..3b0e738781 100644 --- a/boa/src/builtins/boolean/mod.rs +++ b/boa/src/builtins/boolean/mod.rs @@ -97,7 +97,7 @@ pub fn to_boolean(value: &Value) -> Value { match *value.deref().borrow() { ValueData::Object(_) => to_value(true), ValueData::String(ref s) if !s.is_empty() => to_value(true), - ValueData::Number(n) if n != 0.0 && !n.is_nan() => to_value(true), + ValueData::Rational(n) if n != 0.0 && !n.is_nan() => to_value(true), ValueData::Integer(n) if n != 0 => to_value(true), ValueData::Boolean(v) => to_value(v), _ => to_value(false), diff --git a/boa/src/builtins/console/tests.rs b/boa/src/builtins/console/tests.rs index 23085c4b5d..9a3a59fc9c 100644 --- a/boa/src/builtins/console/tests.rs +++ b/boa/src/builtins/console/tests.rs @@ -38,7 +38,7 @@ fn formatter_utf_8_checks() { "Są takie chwile %dą %są tu%sów %привет%ź".to_string(), )), Gc::new(ValueData::Integer(123)), - Gc::new(ValueData::Number(1.23)), + Gc::new(ValueData::Rational(1.23)), Gc::new(ValueData::String("ł".to_string())), ]; let res = formatter(&val); @@ -60,7 +60,7 @@ fn formatter_trailing_format_leader_renders() { fn formatter_float_format_works() { let val = [ Gc::new(ValueData::String("%f".to_string())), - Gc::new(ValueData::Number(3.1415)), + Gc::new(ValueData::Rational(3.1415)), ]; let res = formatter(&val); assert_eq!(res, "3.141500") diff --git a/boa/src/builtins/function/tests.rs b/boa/src/builtins/function/tests.rs index 89801682d4..3efc27fc8e 100644 --- a/boa/src/builtins/function/tests.rs +++ b/boa/src/builtins/function/tests.rs @@ -15,11 +15,11 @@ fn check_arguments_object() { "#; eprintln!("{}", forward(&mut engine, init)); - let expected_return_val: f64 = 100.0; + let expected_return_val = 100; let return_val = forward_val(&mut engine, "val").expect("value expected"); - assert_eq!(return_val.is_double(), true); + assert_eq!(return_val.is_integer(), true); assert_eq!( - from_value::(return_val).expect("Could not convert value to f64"), + from_value::(return_val).expect("Could not convert value to i32"), expected_return_val ); } diff --git a/boa/src/builtins/number/mod.rs b/boa/src/builtins/number/mod.rs index 42841e8eee..4b3b36c0a7 100644 --- a/boa/src/builtins/number/mod.rs +++ b/boa/src/builtins/number/mod.rs @@ -40,7 +40,7 @@ fn to_number(value: &Value) -> Value { ValueData::Integer(i) => to_value(f64::from(i)), ValueData::Object(ref o) => (o).deref().borrow().get_internal_slot("NumberData"), ValueData::Null => to_value(0), - ValueData::Number(n) => to_value(n), + ValueData::Rational(n) => to_value(n), ValueData::String(ref s) => match s.parse::() { Ok(n) => to_value(n), Err(_) => to_value(f64::NAN), diff --git a/boa/src/builtins/object/mod.rs b/boa/src/builtins/object/mod.rs index 850c20fa02..5fd2ec13c9 100644 --- a/boa/src/builtins/object/mod.rs +++ b/boa/src/builtins/object/mod.rs @@ -407,7 +407,7 @@ impl Object { pub fn from(value: &Value) -> Result { match *value.deref().borrow() { ValueData::Boolean(_) => Ok(Self::from_boolean(value)), - ValueData::Number(_) => Ok(Self::from_number(value)), + ValueData::Rational(_) => Ok(Self::from_number(value)), ValueData::String(_) => Ok(Self::from_string(value)), ValueData::Object(ref obj) => Ok(obj.borrow().clone()), _ => Err(()), diff --git a/boa/src/builtins/value/mod.rs b/boa/src/builtins/value/mod.rs index 9bd2a55e4b..ff20598631 100644 --- a/boa/src/builtins/value/mod.rs +++ b/boa/src/builtins/value/mod.rs @@ -48,7 +48,7 @@ pub enum ValueData { /// `String` - A UTF-8 string, such as `"Hello, world"` String(String), /// `Number` - A 64-bit floating point number, such as `3.1415` - Number(f64), + Rational(f64), /// `Number` - A 32-bit integer, such as `42` Integer(i32), /// `Object` - An object, such as `Math`, represented by a binary tree of string keys to Javascript values @@ -150,7 +150,21 @@ impl ValueData { /// Returns true if the value is a 64-bit floating-point number pub fn is_double(&self) -> bool { match *self { - Self::Number(_) => true, + Self::Rational(_) => true, + _ => false, + } + } + + /// Returns true if the value is integer. + #[allow(clippy::float_cmp)] + pub fn is_integer(&self) -> bool { + // If it can fit in a i32 and the trucated version is + // equal to the original then it is an integer. + let is_racional_intiger = |n: f64| n == ((n as i32) as f64); + + match *self { + Self::Integer(_) => true, + Self::Rational(n) if is_racional_intiger(n) => true, _ => false, } } @@ -183,7 +197,7 @@ impl ValueData { match *self { Self::Object(_) => true, Self::String(ref s) if !s.is_empty() => true, - Self::Number(n) if n != 0.0 && !n.is_nan() => true, + Self::Rational(n) if n != 0.0 && !n.is_nan() => true, Self::Integer(n) if n != 0 => true, Self::Boolean(v) => v, _ => false, @@ -198,9 +212,9 @@ impl ValueData { Ok(num) => num, Err(_) => NAN, }, - Self::Number(num) => num, + Self::Rational(num) => num, Self::Boolean(true) => 1.0, - Self::Boolean(false) | Self::Null => 0.0, + Self::Boolean(false) | ValueData::Null => 0.0, Self::Integer(num) => f64::from(num), } } @@ -218,7 +232,7 @@ impl ValueData { Ok(num) => num, Err(_) => 0, }, - Self::Number(num) => num as i32, + Self::Rational(num) => num as i32, Self::Boolean(true) => 1, Self::Integer(num) => num, } @@ -556,7 +570,7 @@ impl ValueData { pub fn from_json(json: JSONValue) -> Self { match json { JSONValue::Number(v) => { - Self::Number(v.as_f64().expect("Could not convert value to f64")) + Self::Rational(v.as_f64().expect("Could not convert value to f64")) } JSONValue::String(v) => Self::String(v), JSONValue::Bool(v) => Self::Boolean(v), @@ -607,7 +621,7 @@ impl ValueData { JSONValue::Object(new_obj) } Self::String(ref str) => JSONValue::String(str.clone()), - Self::Number(num) => JSONValue::Number( + Self::Rational(num) => JSONValue::Number( JSONNumber::from_f64(num).expect("Could not convert to JSONNumber"), ), Self::Integer(val) => JSONValue::Number(JSONNumber::from(val)), @@ -619,7 +633,7 @@ impl ValueData { /// https://tc39.es/ecma262/#sec-typeof-operator pub fn get_type(&self) -> &'static str { match *self { - Self::Number(_) | Self::Integer(_) => "number", + Self::Rational(_) | Self::Integer(_) => "number", Self::String(_) => "string", Self::Boolean(_) => "boolean", Self::Symbol(_) => "symbol", @@ -637,7 +651,7 @@ impl ValueData { } pub fn as_num_to_power(&self, other: Self) -> Self { - Self::Number(self.to_num().powf(other.to_num())) + Self::Rational(self.to_num().powf(other.to_num())) } } @@ -847,7 +861,7 @@ impl Display for ValueData { _ => write!(f, "Symbol()"), }, Self::String(ref v) => write!(f, "{}", v), - Self::Number(v) => write!( + Self::Rational(v) => write!( f, "{}", match v { @@ -884,9 +898,9 @@ impl PartialEq for ValueData { _ if self.is_null_or_undefined() && other.is_null_or_undefined() => true, (Self::String(_), _) | (_, Self::String(_)) => self.to_string() == other.to_string(), (Self::Boolean(a), Self::Boolean(b)) if a == b => true, - (Self::Number(a), Self::Number(b)) if a == b && !a.is_nan() && !b.is_nan() => true, - (Self::Number(a), _) if a == other.to_num() => true, - (_, Self::Number(a)) if a == self.to_num() => true, + (Self::Rational(a), Self::Rational(b)) if a == b && !a.is_nan() && !b.is_nan() => true, + (Self::Rational(a), _) if a == other.to_num() => true, + (_, Self::Rational(a)) if a == self.to_num() => true, (Self::Integer(a), Self::Integer(b)) if a == b => true, _ => false, } @@ -901,32 +915,32 @@ impl Add for ValueData { Self::String(format!("{}{}", s.clone(), &o.to_string())) } (ref s, Self::String(ref o)) => Self::String(format!("{}{}", s.to_string(), o)), - (ref s, ref o) => Self::Number(s.to_num() + o.to_num()), + (ref s, ref o) => Self::Rational(s.to_num() + o.to_num()), } } } impl Sub for ValueData { type Output = Self; fn sub(self, other: Self) -> Self { - Self::Number(self.to_num() - other.to_num()) + Self::Rational(self.to_num() - other.to_num()) } } impl Mul for ValueData { type Output = Self; fn mul(self, other: Self) -> Self { - Self::Number(self.to_num() * other.to_num()) + Self::Rational(self.to_num() * other.to_num()) } } impl Div for ValueData { type Output = Self; fn div(self, other: Self) -> Self { - Self::Number(self.to_num() / other.to_num()) + Self::Rational(self.to_num() / other.to_num()) } } impl Rem for ValueData { type Output = Self; fn rem(self, other: Self) -> Self { - Self::Number(self.to_num() % other.to_num()) + Self::Rational(self.to_num() % other.to_num()) } } impl BitAnd for ValueData { @@ -1027,7 +1041,7 @@ impl FromValue for char { impl ToValue for f64 { fn to_value(&self) -> Value { - Gc::new(ValueData::Number(*self)) + Gc::new(ValueData::Rational(*self)) } } impl FromValue for f64 { diff --git a/boa/src/exec/mod.rs b/boa/src/exec/mod.rs index 9abc9a9181..ce889777b8 100644 --- a/boa/src/exec/mod.rs +++ b/boa/src/exec/mod.rs @@ -536,7 +536,7 @@ impl Executor for Interpreter { ValueData::Symbol(_) => "symbol", ValueData::Null | ValueData::Object(_) => "object", ValueData::Boolean(_) => "boolean", - ValueData::Number(_) | ValueData::Integer(_) => "number", + ValueData::Rational(_) | ValueData::Integer(_) => "number", ValueData::String(_) => "string", ValueData::Function(_) => "function", })) @@ -728,7 +728,7 @@ impl Interpreter { ValueData::Undefined => to_value("undefined"), ValueData::Null => to_value("null"), ValueData::Boolean(ref boolean) => to_value(boolean.to_string()), - ValueData::Number(ref num) => to_value(num.to_string()), + ValueData::Rational(ref num) => to_value(num.to_string()), ValueData::Integer(ref num) => to_value(num.to_string()), ValueData::String(ref string) => to_value(string.clone()), ValueData::Object(_) => { @@ -759,7 +759,7 @@ impl Interpreter { bool_obj.set_internal_slot("BooleanData", value.clone()); Ok(bool_obj) } - ValueData::Number(_) => { + ValueData::Rational(_) => { let proto = self .realm .environment @@ -788,7 +788,7 @@ impl Interpreter { match *value.deref().borrow() { ValueData::Null => String::from("null"), ValueData::Boolean(ref boolean) => boolean.to_string(), - ValueData::Number(ref num) => num.to_string(), + ValueData::Rational(ref num) => num.to_string(), ValueData::Integer(ref num) => num.to_string(), ValueData::String(ref string) => string.clone(), ValueData::Object(_) => { @@ -809,7 +809,7 @@ impl Interpreter { f64::from(0) } } - ValueData::Number(num) => num, + ValueData::Rational(num) => num, ValueData::Integer(num) => f64::from(num), ValueData::String(ref string) => string.parse::().unwrap(), ValueData::Object(_) => { diff --git a/boa/src/syntax/ast/token.rs b/boa/src/syntax/ast/token.rs index 3e706f49ad..0df76ac773 100644 --- a/boa/src/syntax/ast/token.rs +++ b/boa/src/syntax/ast/token.rs @@ -56,6 +56,30 @@ impl Debug for VecToken { } } +/// Represents the type differenct types of numeric literals. +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum NumericLiteral { + /// A floating point number + Rational(f64), + + /// An integer + Integer(i32), + // TODO: Add BigInt +} + +impl From for NumericLiteral { + fn from(n: f64) -> Self { + Self::Rational(n) + } +} + +impl From for NumericLiteral { + fn from(n: i32) -> Self { + Self::Integer(n) + } +} + /// Represents the type of Token and the data it has inside. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, PartialEq, Debug)] @@ -78,7 +102,7 @@ pub enum TokenKind { NullLiteral, /// A numeric literal. - NumericLiteral(f64), + NumericLiteral(NumericLiteral), /// A piece of punctuation /// @@ -138,8 +162,11 @@ impl TokenKind { } /// Creates a `NumericLiteral` token kind. - pub fn numeric_literal(lit: f64) -> Self { - Self::NumericLiteral(lit) + pub fn numeric_literal(lit: L) -> Self + where + L: Into, + { + Self::NumericLiteral(lit.into()) } /// Creates a `Punctuator` token type. @@ -178,7 +205,8 @@ impl Display for TokenKind { Self::Identifier(ref ident) => write!(f, "{}", ident), Self::Keyword(ref word) => write!(f, "{}", word), Self::NullLiteral => write!(f, "null"), - Self::NumericLiteral(ref num) => write!(f, "{}", num), + Self::NumericLiteral(NumericLiteral::Rational(num)) => write!(f, "{}", num), + Self::NumericLiteral(NumericLiteral::Integer(num)) => write!(f, "{}", num), Self::Punctuator(ref punc) => write!(f, "{}", punc), Self::StringLiteral(ref lit) => write!(f, "{}", lit), Self::RegularExpressionLiteral(ref body, ref flags) => write!(f, "/{}/{}", body, flags), diff --git a/boa/src/syntax/lexer/mod.rs b/boa/src/syntax/lexer/mod.rs index 16e277b6af..cda968f4e1 100644 --- a/boa/src/syntax/lexer/mod.rs +++ b/boa/src/syntax/lexer/mod.rs @@ -8,7 +8,7 @@ mod tests; use crate::syntax::ast::{ punc::Punctuator, - token::{Token, TokenKind}, + token::{NumericLiteral, Token, TokenKind}, }; use std::{ char::{decode_utf16, from_u32}, @@ -18,6 +18,7 @@ use std::{ }; /// `vop` tests the next token to see if we're on an assign operation of just a plain binary operation. +/// /// If the next value is not an assignment operation it will pattern match the provided values and return the corresponding token. macro_rules! vop { ($this:ident, $assign_op:expr, $op:expr) => ({ @@ -81,7 +82,7 @@ macro_rules! op { /// [LexerError] implements [fmt::Display] so you just display this value as an error #[derive(Debug, Clone)] pub struct LexerError { - /// details will be displayed when a LexerError occurs + /// details will be displayed when a LexerError occurs. details: String, } @@ -113,7 +114,7 @@ impl error::Error for LexerError { } } -/// A lexical analyzer for JavaScript source code +/// A lexical analyzer for JavaScript source code. #[derive(Debug)] pub struct Lexer<'a> { /// The list of tokens generated so far. @@ -131,9 +132,6 @@ pub struct Lexer<'a> { impl<'a> Lexer<'a> { /// Returns a Lexer with a buffer inside /// - /// # Arguments - /// - /// * `buffer` - A string slice that holds the source code. /// The buffer needs to have a lifetime as long as the Lexer instance itself pub fn new(buffer: &'a str) -> Lexer<'a> { Lexer { @@ -208,23 +206,12 @@ impl<'a> Lexer<'a> { result } - /// Utility function for reading integers in different bases - fn read_integer_in_base(&mut self, base: u32, mut buf: String) -> Result { - self.next(); - while let Some(ch) = self.preview_next() { - if ch.is_digit(base) { - buf.push(self.next()); - } else { - break; - } - } - u64::from_str_radix(&buf, base) - .map_err(|_| LexerError::new("Could not convert value to u64")) - } - - /// Utility function for checkint the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character + /// Utility function for checkint the NumericLiteral is not followed by an `IdentifierStart` or `DecimalDigit` character. /// - /// More info [ECMAScript Specification](https://tc39.es/ecma262/#sec-literals-numeric-literals) + /// More information: + /// - [ECMAScript Specification][spec] + /// + /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals fn check_after_numeric_literal(&mut self) -> Result<(), LexerError> { match self.preview_next() { Some(ch) @@ -237,6 +224,213 @@ impl<'a> Lexer<'a> { } } + /// Lexes a numerical literal. + /// + /// More information: + /// - [ECMAScript Specification][spec] + /// + /// [spec]: https://tc39.es/ecma262/#sec-literals-numeric-literals + fn reed_numerical_literal(&mut self, ch: char) -> Result<(), LexerError> { + /// This is a helper structure + /// + /// This structure helps with identifying what numerical type it is and what base is it. + enum NumericKind { + Rational, + Integer(u32), + BigInt(u32), + } + + impl NumericKind { + /// Get the base of the number kind. + fn base(&self) -> u32 { + match self { + Self::Rational => 10, + Self::Integer(ref base) => *base, + Self::BigInt(ref base) => *base, + } + } + + /// Converts `self` to BigInt kind. + fn convert_to_bigint(&mut self) { + *self = match *self { + Self::Rational => unreachable!("can not convert rational number to BigInt"), + Self::Integer(base) => Self::BigInt(base), + Self::BigInt(base) => Self::BigInt(base), + }; + } + } + + // TODO: Setup strict mode. + let strict_mode = false; + + let mut buf = ch.to_string(); + let mut position_offset = 0; + let mut kind = NumericKind::Integer(10); + if ch == '0' { + match self.preview_next() { + None => { + self.push_token(TokenKind::NumericLiteral(NumericLiteral::Integer(0))); + self.column_number += 1; + return Ok(()); + } + Some('x') | Some('X') => { + self.next(); + position_offset += 1; + kind = NumericKind::Integer(16); + } + Some('o') | Some('O') => { + self.next(); + position_offset += 1; + kind = NumericKind::Integer(8); + } + Some('b') | Some('B') => { + self.next(); + position_offset += 1; + kind = NumericKind::Integer(2); + } + Some(ch) if ch.is_ascii_digit() => { + let mut is_implicit_octal = true; + while let Some(ch) = self.preview_next() { + if !ch.is_ascii_digit() { + break; + } else if !ch.is_digit(8) { + is_implicit_octal = false; + } + buf.push(self.next()); + } + if !strict_mode { + if is_implicit_octal { + kind = NumericKind::Integer(8); + } + } else { + return Err(if is_implicit_octal { + LexerError::new( + "Implicit octal literals are not allowed in strict mode.", + ) + } else { + LexerError::new( + "Decimals with leading zeros are not allowed in strict mode.", + ) + }); + } + } + Some(_) => {} + } + } + + while let Some(ch) = self.preview_next() { + if !ch.is_digit(kind.base()) { + break; + } + buf.push(self.next()); + } + + if self.next_is('n') { + kind.convert_to_bigint() + } + + if let NumericKind::Integer(10) = kind { + 'digitloop: while let Some(ch) = self.preview_next() { + match ch { + '.' => loop { + kind = NumericKind::Rational; + buf.push(self.next()); + + let c = match self.preview_next() { + Some(ch) => ch, + None => break, + }; + + match c { + 'e' | 'E' => { + match self + .preview_multiple_next(2) + .unwrap_or_default() + .to_digit(10) + { + Some(0..=9) | None => { + buf.push(self.next()); + } + _ => { + break 'digitloop; + } + } + } + _ => { + if !c.is_digit(10) { + break 'digitloop; + } + } + } + }, + 'e' | 'E' => { + kind = NumericKind::Rational; + match self + .preview_multiple_next(2) + .unwrap_or_default() + .to_digit(10) + { + Some(0..=9) | None => { + buf.push(self.next()); + } + _ => { + break; + } + } + buf.push(self.next()); + } + '+' | '-' => { + break; + } + _ if ch.is_digit(10) => { + buf.push(self.next()); + } + _ => break, + } + } + } + + if let Err(e) = self.check_after_numeric_literal() { + return Err(e); + }; + + let num = match kind { + NumericKind::BigInt(_) => { + // TODO: Implement bigint. + // NOTE: implementation goes here. + unimplemented!("BigInt"); + } + NumericKind::Rational /* base: 10 */ => { + NumericLiteral::Rational( + f64::from_str(&buf) + .map_err(|_| LexerError::new("Could not convert value to f64"))?, + ) + } + NumericKind::Integer(base) => { + if let Ok(num) = i32::from_str_radix(&buf, base) { + NumericLiteral::Integer( + num + ) + } else { + let b = f64::from(base); + let mut result = 0.0_f64; + for c in buf.chars() { + let digit = f64::from(c.to_digit(base).unwrap()); + result = result * b + digit; + } + + NumericLiteral::Rational(result) + } + + } + }; + + self.push_token(TokenKind::NumericLiteral(num)); + self.column_number += (buf.len() as u64) + position_offset - 1; + + Ok(()) + } + /// Runs the lexer until completion, returning a [LexerError] if there's a syntax issue, or an empty unit result /// /// # Example @@ -383,117 +577,7 @@ impl<'a> Lexer<'a> { // to compensate for the incrementing at the top self.column_number += str_length.wrapping_add(1); } - '0' => { - let mut buf = String::new(); - - let num = match self.preview_next() { - None => { - self.push_token(TokenKind::NumericLiteral(0_f64)); - return Ok(()); - } - Some('x') | Some('X') => { - self.read_integer_in_base(16, buf)? as f64 - } - Some('o') | Some('O') => { - self.read_integer_in_base(8, buf)? as f64 - } - Some('b') | Some('B') => { - self.read_integer_in_base(2, buf)? as f64 - } - Some(ch) if (ch.is_ascii_digit() || ch == '.') => { - // LEGACY OCTAL (ONLY FOR NON-STRICT MODE) - let mut gone_decimal = ch == '.'; - while let Some(next_ch) = self.preview_next() { - match next_ch { - c if next_ch.is_digit(8) => { - buf.push(c); - self.next(); - } - '8' | '9' | '.' => { - gone_decimal = true; - buf.push(next_ch); - self.next(); - } - _ => { - break; - } - } - } - if gone_decimal { - f64::from_str(&buf).map_err(|_e| LexerError::new("Could not convert value to f64"))? - } else if buf.is_empty() { - 0.0 - } else { - (u64::from_str_radix(&buf, 8).map_err(|_e| LexerError::new("Could not convert value to u64"))?) as f64 - } - } - Some(_) => { - 0.0 - } - }; - - self.push_token(TokenKind::NumericLiteral(num)); - - //11.8.3 - if let Err(e) = self.check_after_numeric_literal() { - return Err(e) - }; - } - _ if ch.is_digit(10) => { - let mut buf = ch.to_string(); - 'digitloop: while let Some(ch) = self.preview_next() { - match ch { - '.' => loop { - buf.push(self.next()); - - let c = match self.preview_next() { - Some(ch) => ch, - None => break, - }; - - match c { - 'e' | 'E' => { - match self.preview_multiple_next(2).unwrap_or_default().to_digit(10) { - Some(0..=9) | None => { - buf.push(self.next()); - } - _ => { - break 'digitloop; - } - } - } - _ => { - if !c.is_digit(10) { - break 'digitloop; - } - } - } - }, - 'e' | 'E' => { - match self.preview_multiple_next(2).unwrap_or_default().to_digit(10) { - Some(0..=9) | None => { - buf.push(self.next()); - } - _ => { - break; - } - } - buf.push(self.next()); - } - '+' | '-' => { - break; - } - _ if ch.is_digit(10) => { - buf.push(self.next()); - } - _ => break, - } - } - // TODO make this a bit more safe -------------------------------VVVV - self.push_token(TokenKind::NumericLiteral( - f64::from_str(&buf).map_err(|_| LexerError::new("Could not convert value to f64"))?, - )) - } + _ if ch.is_digit(10) => self.reed_numerical_literal(ch)?, _ if ch.is_alphabetic() || ch == '$' || ch == '_' => { let mut buf = ch.to_string(); while let Some(ch) = self.preview_next() { diff --git a/boa/src/syntax/lexer/tests.rs b/boa/src/syntax/lexer/tests.rs index 42146b593a..907a3f877b 100644 --- a/boa/src/syntax/lexer/tests.rs +++ b/boa/src/syntax/lexer/tests.rs @@ -376,24 +376,51 @@ fn numbers() { ); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1.0)); - assert_eq!(lexer.tokens[1].kind, TokenKind::NumericLiteral(2.0)); - assert_eq!(lexer.tokens[2].kind, TokenKind::NumericLiteral(52.0)); - assert_eq!(lexer.tokens[3].kind, TokenKind::NumericLiteral(46.0)); - assert_eq!(lexer.tokens[4].kind, TokenKind::NumericLiteral(7.89)); - assert_eq!(lexer.tokens[5].kind, TokenKind::NumericLiteral(42.0)); - assert_eq!(lexer.tokens[6].kind, TokenKind::NumericLiteral(5000.0)); - assert_eq!(lexer.tokens[7].kind, TokenKind::NumericLiteral(5000.0)); - assert_eq!(lexer.tokens[8].kind, TokenKind::NumericLiteral(0.005)); - assert_eq!(lexer.tokens[9].kind, TokenKind::NumericLiteral(2.0)); - assert_eq!(lexer.tokens[10].kind, TokenKind::NumericLiteral(83.0)); - assert_eq!(lexer.tokens[11].kind, TokenKind::NumericLiteral(999.0)); - assert_eq!(lexer.tokens[12].kind, TokenKind::NumericLiteral(10.0)); - assert_eq!(lexer.tokens[13].kind, TokenKind::NumericLiteral(0.1)); - assert_eq!(lexer.tokens[14].kind, TokenKind::NumericLiteral(10.0)); - assert_eq!(lexer.tokens[15].kind, TokenKind::NumericLiteral(10.0)); - assert_eq!(lexer.tokens[16].kind, TokenKind::NumericLiteral(0.0)); - assert_eq!(lexer.tokens[17].kind, TokenKind::NumericLiteral(0.12)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); + assert_eq!(lexer.tokens[1].kind, TokenKind::numeric_literal(2)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(52)); + assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(46)); + assert_eq!(lexer.tokens[4].kind, TokenKind::numeric_literal(7.89)); + assert_eq!(lexer.tokens[5].kind, TokenKind::numeric_literal(42.0)); + assert_eq!(lexer.tokens[6].kind, TokenKind::numeric_literal(5000.0)); + assert_eq!(lexer.tokens[7].kind, TokenKind::numeric_literal(5000.0)); + assert_eq!(lexer.tokens[8].kind, TokenKind::numeric_literal(0.005)); + assert_eq!(lexer.tokens[9].kind, TokenKind::numeric_literal(2)); + assert_eq!(lexer.tokens[10].kind, TokenKind::numeric_literal(83)); + assert_eq!(lexer.tokens[11].kind, TokenKind::numeric_literal(999)); + assert_eq!(lexer.tokens[12].kind, TokenKind::numeric_literal(10.0)); + assert_eq!(lexer.tokens[13].kind, TokenKind::numeric_literal(0.1)); + assert_eq!(lexer.tokens[14].kind, TokenKind::numeric_literal(10.0)); + assert_eq!(lexer.tokens[15].kind, TokenKind::numeric_literal(10.0)); + assert_eq!(lexer.tokens[16].kind, TokenKind::numeric_literal(0.0)); + assert_eq!(lexer.tokens[17].kind, TokenKind::numeric_literal(0.12)); +} + +#[test] +fn implicit_octal_edge_case() { + let mut lexer = Lexer::new("044.5 094.5"); + + lexer.lex().expect("failed to lex"); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(36)); + assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(5)); + + assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(94.5)); +} + +#[test] +fn hexadecimal_edge_case() { + let mut lexer = Lexer::new("0xffff.ff 0xffffff"); + + lexer.lex().expect("failed to lex"); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(0xffff)); + assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot)); + assert_eq!( + lexer.tokens[2].kind, + TokenKind::Identifier(String::from("ff")) + ); + + assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(0xffffff)); } #[test] @@ -406,7 +433,7 @@ fn test_single_number_without_semicolon() { fn test_number_followed_by_dot() { let mut lexer = Lexer::new("1.."); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1.0)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot)); } @@ -434,55 +461,55 @@ fn test_regex_literal_flags() { fn test_addition_no_spaces() { let mut lexer = Lexer::new("1+1"); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); } #[test] fn test_addition_no_spaces_left_side() { let mut lexer = Lexer::new("1+ 1"); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); } #[test] fn test_addition_no_spaces_right_side() { let mut lexer = Lexer::new("1 +1"); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); } #[test] fn test_addition_no_spaces_e_number_left_side() { let mut lexer = Lexer::new("1e2+ 1"); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(100.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(100.0)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1)); } #[test] fn test_addition_no_spaces_e_number_right_side() { let mut lexer = Lexer::new("1 +1e3"); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); - assert_eq!(lexer.tokens[2].kind, TokenKind::NumericLiteral(1000.0)); + assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1000.0)); } #[test] fn test_addition_no_spaces_e_number() { let mut lexer = Lexer::new("1e3+1e11"); lexer.lex().expect("failed to lex"); - assert_eq!(lexer.tokens[0].kind, TokenKind::NumericLiteral(1000.0)); + assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1000.0)); assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add)); assert_eq!( lexer.tokens[2].kind, - TokenKind::NumericLiteral(100_000_000_000.0) + TokenKind::numeric_literal(100_000_000_000.0) ); } diff --git a/boa/src/syntax/parser/expression/primary/array_initializer/tests.rs b/boa/src/syntax/parser/expression/primary/array_initializer/tests.rs index 5345616459..ec15a3d946 100644 --- a/boa/src/syntax/parser/expression/primary/array_initializer/tests.rs +++ b/boa/src/syntax/parser/expression/primary/array_initializer/tests.rs @@ -26,9 +26,9 @@ fn check_numeric_array() { check_parser( "[1, 2, 3]", &[Node::ArrayDecl(vec![ - Node::const_node(1.0), - Node::const_node(2.0), - Node::const_node(3.0), + Node::const_node(1), + Node::const_node(2), + Node::const_node(3), ])], ); } @@ -39,9 +39,9 @@ fn check_numeric_array_trailing() { check_parser( "[1, 2, 3,]", &[Node::ArrayDecl(vec![ - Node::const_node(1.0), - Node::const_node(2.0), - Node::const_node(3.0), + Node::const_node(1), + Node::const_node(2), + Node::const_node(3), ])], ); } @@ -52,10 +52,10 @@ fn check_numeric_array_elision() { check_parser( "[1, 2, , 3]", &[Node::ArrayDecl(vec![ - Node::const_node(1.0), - Node::const_node(2.0), + Node::const_node(1), + Node::const_node(2), Node::Const(Const::Undefined), - Node::const_node(3.0), + Node::const_node(3), ])], ); } @@ -66,11 +66,11 @@ fn check_numeric_array_repeated_elision() { check_parser( "[1, 2, ,, 3]", &[Node::ArrayDecl(vec![ - Node::const_node(1.0), - Node::const_node(2.0), + Node::const_node(1), + Node::const_node(2), Node::Const(Const::Undefined), Node::Const(Const::Undefined), - Node::const_node(3.0), + Node::const_node(3), ])], ); } @@ -81,9 +81,9 @@ fn check_combined() { check_parser( "[1, \"a\", 2]", &[Node::ArrayDecl(vec![ - Node::const_node(1.0), + Node::const_node(1), Node::const_node("a"), - Node::const_node(2.0), + Node::const_node(2), ])], ); } @@ -94,9 +94,9 @@ fn check_combined_empty_str() { check_parser( "[1, \"\", 2]", &[Node::ArrayDecl(vec![ - Node::const_node(1.0), + Node::const_node(1), Node::const_node(""), - Node::const_node(2.0), + Node::const_node(2), ])], ); } diff --git a/boa/src/syntax/parser/expression/primary/mod.rs b/boa/src/syntax/parser/expression/primary/mod.rs index 4b2fcc0cca..2cb684b3cf 100644 --- a/boa/src/syntax/parser/expression/primary/mod.rs +++ b/boa/src/syntax/parser/expression/primary/mod.rs @@ -19,7 +19,10 @@ use self::{ }; use super::Expression; use crate::syntax::{ - ast::{constant::Const, keyword::Keyword, node::Node, punc::Punctuator, token::TokenKind}, + ast::{ + constant::Const, keyword::Keyword, node::Node, punc::Punctuator, token::NumericLiteral, + token::TokenKind, + }, parser::{AllowAwait, AllowYield, Cursor, ParseError, ParseResult, TokenParser}, }; pub(in crate::syntax::parser) use object_initializer::Initializer; @@ -80,7 +83,8 @@ impl TokenParser for PrimaryExpression { TokenKind::NullLiteral => Ok(Node::Const(Const::Null)), TokenKind::Identifier(ident) => Ok(Node::local(ident)), TokenKind::StringLiteral(s) => Ok(Node::const_node(s)), - TokenKind::NumericLiteral(num) => Ok(Node::const_node(*num)), + TokenKind::NumericLiteral(NumericLiteral::Integer(num)) => Ok(Node::const_node(*num)), + TokenKind::NumericLiteral(NumericLiteral::Rational(num)) => Ok(Node::const_node(*num)), TokenKind::RegularExpressionLiteral(body, flags) => Ok(Node::new(Node::call( Node::local("RegExp"), vec![Node::const_node(body), Node::const_node(flags)], diff --git a/boa/src/syntax/parser/expression/tests.rs b/boa/src/syntax/parser/expression/tests.rs index 5e6819cce4..3b104ec885 100644 --- a/boa/src/syntax/parser/expression/tests.rs +++ b/boa/src/syntax/parser/expression/tests.rs @@ -16,7 +16,7 @@ fn check_numeric_operations() { &[Node::bin_op( NumOp::Add, Node::local("a"), - Node::const_node(1.0), + Node::const_node(1), )], ); check_parser( @@ -28,7 +28,7 @@ fn check_numeric_operations() { &[Node::bin_op( NumOp::Sub, Node::local("a"), - Node::const_node(1.0), + Node::const_node(1), )], ); check_parser( @@ -40,7 +40,7 @@ fn check_numeric_operations() { &[Node::bin_op( NumOp::Div, Node::local("a"), - Node::const_node(2.0), + Node::const_node(2), )], ); check_parser( @@ -52,7 +52,7 @@ fn check_numeric_operations() { &[Node::bin_op( NumOp::Mul, Node::local("a"), - Node::const_node(2.0), + Node::const_node(2), )], ); check_parser( @@ -64,7 +64,7 @@ fn check_numeric_operations() { &[Node::bin_op( NumOp::Exp, Node::local("a"), - Node::const_node(2.0), + Node::const_node(2), )], ); check_parser( @@ -76,7 +76,7 @@ fn check_numeric_operations() { &[Node::bin_op( NumOp::Mod, Node::local("a"), - Node::const_node(2.0), + Node::const_node(2), )], ); } @@ -94,10 +94,10 @@ fn check_complex_numeric_operations() { Node::bin_op( NumOp::Mul, Node::local("d"), - Node::bin_op(NumOp::Sub, Node::local("b"), Node::const_node(3.0)), + Node::bin_op(NumOp::Sub, Node::local("b"), Node::const_node(3)), ), ), - Node::const_node(1.0), + Node::const_node(1), )], ); } @@ -287,7 +287,7 @@ fn check_assign_operations() { &[Node::bin_op( BinOp::Assign(AssignOp::Mod), Node::local("a"), - Node::bin_op(NumOp::Div, Node::const_node(10.0), Node::const_node(2.0)), + Node::bin_op(NumOp::Div, Node::const_node(10), Node::const_node(2)), )], ); } diff --git a/boa/src/syntax/parser/statement/declaration/tests.rs b/boa/src/syntax/parser/statement/declaration/tests.rs index b4433d6b93..99200e0028 100644 --- a/boa/src/syntax/parser/statement/declaration/tests.rs +++ b/boa/src/syntax/parser/statement/declaration/tests.rs @@ -10,7 +10,7 @@ fn check_var_declaration() { "var a = 5;", &[Node::VarDecl(vec![( String::from("a"), - Some(Node::const_node(5.0)), + Some(Node::const_node(5)), )])], ); } @@ -22,7 +22,7 @@ fn check_var_declaration_no_spaces() { "var a=5;", &[Node::VarDecl(vec![( String::from("a"), - Some(Node::const_node(5.0)), + Some(Node::const_node(5)), )])], ); } @@ -39,9 +39,9 @@ fn check_multiple_var_declaration() { check_parser( "var a = 5, b, c = 6;", &[Node::VarDecl(vec![ - (String::from("a"), Some(Node::const_node(5.0))), + (String::from("a"), Some(Node::const_node(5))), (String::from("b"), None), - (String::from("c"), Some(Node::const_node(6.0))), + (String::from("c"), Some(Node::const_node(6))), ])], ); } @@ -53,7 +53,7 @@ fn check_let_declaration() { "let a = 5;", &[Node::LetDecl(vec![( String::from("a"), - Some(Node::const_node(5.0)), + Some(Node::const_node(5)), )])], ); } @@ -65,7 +65,7 @@ fn check_let_declaration_no_spaces() { "let a=5;", &[Node::LetDecl(vec![( String::from("a"), - Some(Node::const_node(5.0)), + Some(Node::const_node(5)), )])], ); } @@ -82,9 +82,9 @@ fn check_multiple_let_declaration() { check_parser( "let a = 5, b, c = 6;", &[Node::LetDecl(vec![ - (String::from("a"), Some(Node::const_node(5.0))), + (String::from("a"), Some(Node::const_node(5))), (String::from("b"), None), - (String::from("c"), Some(Node::const_node(6.0))), + (String::from("c"), Some(Node::const_node(6))), ])], ); } @@ -96,7 +96,7 @@ fn check_const_declaration() { "const a = 5;", &[Node::ConstDecl(vec![( String::from("a"), - Node::const_node(5.0), + Node::const_node(5), )])], ); } @@ -108,7 +108,7 @@ fn check_const_declaration_no_spaces() { "const a=5;", &[Node::ConstDecl(vec![( String::from("a"), - Node::const_node(5.0), + Node::const_node(5), )])], ); } @@ -125,8 +125,8 @@ fn check_multiple_const_declaration() { check_parser( "const a = 5, c = 6;", &[Node::ConstDecl(vec![ - (String::from("a"), Node::const_node(5.0)), - (String::from("c"), Node::const_node(6.0)), + (String::from("a"), Node::const_node(5)), + (String::from("c"), Node::const_node(6)), ])], ); } diff --git a/boa/src/syntax/parser/statement/iteration/tests.rs b/boa/src/syntax/parser/statement/iteration/tests.rs index e71dab5d6c..06fc5edd98 100644 --- a/boa/src/syntax/parser/statement/iteration/tests.rs +++ b/boa/src/syntax/parser/statement/iteration/tests.rs @@ -15,7 +15,7 @@ fn check_do_while() { Node::Block(vec![Node::bin_op( BinOp::Assign(AssignOp::Add), Node::local("a"), - Node::const_node(1.0), + Node::const_node(1), )]), Node::const_node(true), )], @@ -29,7 +29,7 @@ fn check_do_while_semicolon_insertion() { r#"var i = 0; do {console.log("hello");} while(i++ < 10) console.log("end");"#, &[ - Node::VarDecl(vec![(String::from("i"), Some(Node::const_node(0.0)))]), + Node::VarDecl(vec![(String::from("i"), Some(Node::const_node(0)))]), Node::do_while_loop( Node::Block(vec![Node::call( Node::get_const_field(Node::local("console"), "log"), @@ -38,7 +38,7 @@ fn check_do_while_semicolon_insertion() { Node::bin_op( BinOp::Comp(CompOp::LessThan), Node::unary_op(UnaryOp::IncrementPost, Node::local("i")), - Node::const_node(10.0), + Node::const_node(10), ), ), Node::call( diff --git a/boa/src/syntax/parser/tests.rs b/boa/src/syntax/parser/tests.rs index 7b14566bdd..088f63ebd8 100644 --- a/boa/src/syntax/parser/tests.rs +++ b/boa/src/syntax/parser/tests.rs @@ -44,7 +44,7 @@ fn assing_operator_precedence() { "a = a + 1", &[Node::assign( Node::local("a"), - Node::bin_op(NumOp::Add, Node::local("a"), Node::const_node(1.0)), + Node::bin_op(NumOp::Add, Node::local("a"), Node::const_node(1)), )], ); }