diff --git a/Cargo.lock b/Cargo.lock index 7a6d9cafda..ecff1290e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "0.7.19" @@ -26,12 +41,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "anyhow" -version = "1.0.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" - [[package]] name = "arbitrary" version = "1.2.0" @@ -58,6 +67,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -210,13 +234,13 @@ dependencies = [ name = "boa_tester" version = "0.16.0" dependencies = [ - "anyhow", "bitflags", "boa_engine", "boa_gc", "boa_interner", "boa_parser", "clap 4.0.25", + "color-eyre", "colored", "fxhash", "once_cell", @@ -225,6 +249,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "toml", ] [[package]] @@ -394,6 +419,33 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "color-eyre" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + [[package]] name = "colored" version = "2.0.0" @@ -626,6 +678,16 @@ dependencies = [ "str-buf", ] +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fast-float" version = "0.2.0" @@ -692,6 +754,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" + [[package]] name = "half" version = "1.8.2" @@ -872,6 +940,12 @@ dependencies = [ "icu_provider_blob", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "1.9.1" @@ -1036,6 +1110,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -1098,6 +1181,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.16.0" @@ -1116,6 +1208,12 @@ version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parking_lot" version = "0.11.2" @@ -1192,6 +1290,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + [[package]] name = "plotters" version = "0.3.4" @@ -1394,6 +1498,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1525,6 +1635,15 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "siphasher" version = "0.3.10" @@ -1647,6 +1766,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "time" version = "0.1.44" @@ -1695,6 +1823,57 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + [[package]] name = "unicode-general-category" version = "0.6.0" @@ -1746,6 +1925,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.4" diff --git a/boa_tester/Cargo.toml b/boa_tester/Cargo.toml index 3871a5817c..9df9f58cd3 100644 --- a/boa_tester/Cargo.toml +++ b/boa_tester/Cargo.toml @@ -26,4 +26,5 @@ once_cell = "1.16.0" colored = "2.0.0" fxhash = "0.2.1" rayon = "1.5.3" -anyhow = "1.0.66" +toml = "0.5.9" +color-eyre = "0.6.2" diff --git a/boa_tester/src/exec/mod.rs b/boa_tester/src/exec/mod.rs index dc94889464..254bebc0fa 100644 --- a/boa_tester/src/exec/mod.rs +++ b/boa_tester/src/exec/mod.rs @@ -2,11 +2,12 @@ mod js262; +use std::borrow::Cow; + use crate::read::ErrorType; use super::{ - Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, - TestSuite, IGNORED, + Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, TestSuite, }; use boa_engine::{ builtins::JsArgs, object::FunctionBuilder, property::Attribute, Context, JsNativeErrorKind, @@ -130,6 +131,24 @@ impl Test { /// Runs the test once, in strict or non-strict mode fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult { + if self.ignored { + if verbose > 1 { + println!( + "`{}`{}: {}", + self.name, + if strict { " (strict mode)" } else { "" }, + "Ignored".yellow() + ); + } else { + print!("{}", "-".yellow()); + } + return TestResult { + name: self.name.clone(), + strict, + result: TestOutcomeResult::Ignored, + result_text: Box::default(), + }; + } if verbose > 1 { println!( "`{}`{}: starting", @@ -139,183 +158,143 @@ impl Test { } let test_content = if strict { - format!("\"use strict\";\n{}", self.content) + Cow::Owned(format!("\"use strict\";\n{}", self.content)) } else { - self.content.to_string() + Cow::Borrowed(&*self.content) }; - let (result, result_text) = if !IGNORED.contains_any_flag(self.flags) - && !IGNORED.contains_test(&self.name) - && !IGNORED.contains_any_feature(&self.features) - && (matches!(self.expected_outcome, Outcome::Positive) - || matches!( - self.expected_outcome, - Outcome::Negative { - phase: Phase::Parse, - error_type: _, - } - ) - || matches!( - self.expected_outcome, - Outcome::Negative { - phase: Phase::Early, - error_type: _, - } - ) - || matches!( - self.expected_outcome, - Outcome::Negative { - phase: Phase::Runtime, - error_type: _, - } - )) { - let res = std::panic::catch_unwind(|| match self.expected_outcome { - Outcome::Positive => { - let mut context = Context::default(); - let async_result = AsyncResult::default(); - - if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) { - return (false, e); - } + let result = std::panic::catch_unwind(|| match self.expected_outcome { + Outcome::Positive => { + let mut context = Context::default(); + let async_result = AsyncResult::default(); - // TODO: timeout - let value = match context.eval(&test_content) { - Ok(v) => v, - Err(e) => return (false, format!("Uncaught {e}")), - }; + if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) { + return (false, e); + } - if let Err(e) = async_result.inner.borrow().as_ref() { - return (false, format!("Uncaught {e}")); - } + // TODO: timeout + let value = match context.eval(&*test_content) { + Ok(v) => v, + Err(e) => return (false, format!("Uncaught {e}")), + }; - (true, value.display().to_string()) + if let Err(e) = async_result.inner.borrow().as_ref() { + return (false, format!("Uncaught {e}")); } - Outcome::Negative { - phase: Phase::Parse | Phase::Early, + + (true, value.display().to_string()) + } + Outcome::Negative { + phase: Phase::Parse | Phase::Early, + error_type, + } => { + assert_eq!( error_type, - } => { - assert_eq!( - error_type, - ErrorType::SyntaxError, - "non-SyntaxError parsing/early error found in {}", - self.name - ); - - let mut context = Context::default(); - match context.parse(&test_content) { - Ok(statement_list) => match context.compile(&statement_list) { - Ok(_) => (false, "StatementList compilation should fail".to_owned()), - Err(e) => (true, format!("Uncaught {e:?}")), - }, - Err(e) => (true, format!("Uncaught {e}")), - } + ErrorType::SyntaxError, + "non-SyntaxError parsing/early error found in {}", + self.name + ); + + let mut context = Context::default(); + match context.parse(&*test_content) { + Ok(statement_list) => match context.compile(&statement_list) { + Ok(_) => (false, "StatementList compilation should fail".to_owned()), + Err(e) => (true, format!("Uncaught {e:?}")), + }, + Err(e) => (true, format!("Uncaught {e}")), } - Outcome::Negative { - phase: Phase::Resolution, - error_type: _, - } => todo!("check module resolution errors"), - Outcome::Negative { - phase: Phase::Runtime, - error_type, - } => { - let mut context = Context::default(); - if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) { - return (false, e); - } - let code = match Parser::new(test_content.as_bytes()) - .parse_all(context.interner_mut()) - .map_err(Into::into) - .and_then(|stmts| context.compile(&stmts)) - { - Ok(code) => code, - Err(e) => return (false, format!("Uncaught {e}")), - }; - - // TODO: timeout - let e = match context.execute(code) { - Ok(res) => return (false, res.display().to_string()), - Err(e) => e, - }; - if let Ok(e) = e.try_native(&mut context) { - match &e.kind { - JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {} - JsNativeErrorKind::Reference - if error_type == ErrorType::ReferenceError => {} - JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {} - JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {} - _ => return (false, format!("Uncaught {e}")), + } + Outcome::Negative { + phase: Phase::Resolution, + error_type: _, + } => todo!("check module resolution errors"), + Outcome::Negative { + phase: Phase::Runtime, + error_type, + } => { + let mut context = Context::default(); + if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) { + return (false, e); + } + let code = match Parser::new(test_content.as_bytes()) + .parse_all(context.interner_mut()) + .map_err(Into::into) + .and_then(|stmts| context.compile(&stmts)) + { + Ok(code) => code, + Err(e) => return (false, format!("Uncaught {e}")), + }; + + // TODO: timeout + let e = match context.execute(code) { + Ok(res) => return (false, res.display().to_string()), + Err(e) => e, + }; + if let Ok(e) = e.try_native(&mut context) { + match &e.kind { + JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {} + JsNativeErrorKind::Reference if error_type == ErrorType::ReferenceError => { } - (true, format!("Uncaught {e}")) - } else { - let passed = e - .as_opaque() - .expect("try_native cannot fail if e is not opaque") - .as_object() - .and_then(|o| o.get("constructor", &mut context).ok()) - .as_ref() - .and_then(JsValue::as_object) - .and_then(|o| o.get("name", &mut context).ok()) - .as_ref() - .and_then(JsValue::as_string) - .map(|s| s == error_type.as_str()) - .unwrap_or_default(); - (passed, format!("Uncaught {e}")) + JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {} + JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {} + _ => return (false, format!("Uncaught {e}")), } + (true, format!("Uncaught {e}")) + } else { + let passed = e + .as_opaque() + .expect("try_native cannot fail if e is not opaque") + .as_object() + .and_then(|o| o.get("constructor", &mut context).ok()) + .as_ref() + .and_then(JsValue::as_object) + .and_then(|o| o.get("name", &mut context).ok()) + .as_ref() + .and_then(JsValue::as_string) + .map(|s| s == error_type.as_str()) + .unwrap_or_default(); + (passed, format!("Uncaught {e}")) } - }); - - let result = res.map_or_else( - |_| { - eprintln!("last panic was on test \"{}\"", self.name); - (TestOutcomeResult::Panic, String::new()) - }, - |(res, text)| { - if res { - (TestOutcomeResult::Passed, text) - } else { - (TestOutcomeResult::Failed, text) - } - }, - ); - - if verbose > 1 { - println!( - "`{}`{}: {}", - self.name, - if strict { " (strict mode)" } else { "" }, - if matches!(result, (TestOutcomeResult::Passed, _)) { - "Passed".green() - } else if matches!(result, (TestOutcomeResult::Failed, _)) { - "Failed".red() - } else { - "⚠ Panic ⚠".red() - } - ); - } else { - print!( - "{}", - if matches!(result, (TestOutcomeResult::Passed, _)) { - ".".green() - } else { - "F".red() - } - ); } + }); + + let (result, result_text) = result.map_or_else( + |_| { + eprintln!("last panic was on test \"{}\"", self.name); + (TestOutcomeResult::Panic, String::new()) + }, + |(res, text)| { + if res { + (TestOutcomeResult::Passed, text) + } else { + (TestOutcomeResult::Failed, text) + } + }, + ); - result + if verbose > 1 { + println!( + "`{}`{}: {}", + self.name, + if strict { " (strict mode)" } else { "" }, + if result == TestOutcomeResult::Passed { + "Passed".green() + } else if result == TestOutcomeResult::Failed { + "Failed".red() + } else { + "⚠ Panic ⚠".red() + } + ); } else { - if verbose > 1 { - println!( - "`{}`{}: {}", - self.name, - if strict { " (strict mode)" } else { "" }, - "Ignored".yellow() - ); - } else { - print!("{}", "-".yellow()); - } - (TestOutcomeResult::Ignored, String::new()) - }; + print!( + "{}", + if result == TestOutcomeResult::Passed { + ".".green() + } else { + "F".red() + } + ); + } if verbose > 2 { println!( diff --git a/boa_tester/src/main.rs b/boa_tester/src/main.rs index 54ef04d9a4..41b7b0f584 100644 --- a/boa_tester/src/main.rs +++ b/boa_tester/src/main.rs @@ -1,7 +1,7 @@ //! Test262 test runner //! //! This crate will run the full ECMAScript test suite (Test262) and report compliance of the -//! `boa` context. +//! `boa` wrap_err. #![doc( html_logo_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg", html_favicon_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg" @@ -70,26 +70,34 @@ use self::{ read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag}, results::{compare_results, write_json}, }; -use anyhow::{bail, Context}; use bitflags::bitflags; use clap::{ArgAction, Parser, ValueHint}; +use color_eyre::{ + eyre::{bail, WrapErr}, + Result, +}; use colored::Colorize; use fxhash::{FxHashMap, FxHashSet}; -use once_cell::sync::Lazy; use read::ErrorType; -use serde::{Deserialize, Serialize}; +use serde::{ + de::{Unexpected, Visitor}, + Deserialize, Deserializer, Serialize, +}; use std::{ - fs, + fs::{self, File}, + io::Read, path::{Path, PathBuf}, }; /// Structure to allow defining ignored tests, features and files that should /// be ignored even when reading. -#[derive(Debug)] +#[derive(Debug, Deserialize)] struct Ignored { + #[serde(default)] tests: FxHashSet>, + #[serde(default)] features: FxHashSet>, - files: FxHashSet>, + #[serde(default = "TestFlags::empty")] flags: TestFlags, } @@ -102,16 +110,17 @@ impl Ignored { /// Checks if the ignore list contains the given feature name in the list /// of features to ignore. - pub(crate) fn contains_any_feature(&self, features: &[Box]) -> bool { - features - .iter() - .any(|feature| self.features.contains(feature)) - } - - /// Checks if the ignore list contains the given file name in the list to - /// ignore from reading. - pub(crate) fn contains_file(&self, file: &str) -> bool { - self.files.contains(file) + pub(crate) fn contains_feature(&self, feature: &str) -> bool { + if self.features.contains(feature) { + return true; + } + // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. + // This ensures those are also ignored. + feature + .split('.') + .next() + .map(|feat| self.features.contains(feat)) + .unwrap_or_default() } pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool { @@ -124,74 +133,11 @@ impl Default for Ignored { Self { tests: FxHashSet::default(), features: FxHashSet::default(), - files: FxHashSet::default(), flags: TestFlags::empty(), } } } -/// List of ignored tests. -static IGNORED: Lazy = Lazy::new(|| { - let path = Path::new("test_ignore.txt"); - if path.exists() { - let filtered = fs::read_to_string(path).expect("could not read test filters"); - filtered - .lines() - .filter(|line| !line.is_empty() && !line.starts_with("//")) - .fold(Ignored::default(), |mut ign, line| { - // let mut line = line.to_owned(); - if line.starts_with("file:") { - let file = line - .strip_prefix("file:") - .expect("prefix disappeared") - .trim() - .to_owned() - .into_boxed_str(); - let test = if file.ends_with(".js") { - file.strip_suffix(".js") - .expect("suffix disappeared") - .to_owned() - .into_boxed_str() - } else { - file.clone() - }; - ign.files.insert(file); - ign.tests.insert(test); - } else if line.starts_with("feature:") { - ign.features.insert( - line.strip_prefix("feature:") - .expect("prefix disappeared") - .trim() - .to_owned() - .into_boxed_str(), - ); - } else if line.starts_with("flag:") { - let flag = line - .strip_prefix("flag:") - .expect("prefix disappeared") - .trim() - .parse::() - .expect("invalid flag found"); - ign.flags.insert(flag.into()); - } else { - let mut test = line.trim(); - if test - .rsplit('.') - .next() - .map(|ext| ext.eq_ignore_ascii_case("js")) - == Some(true) - { - test = test.strip_suffix(".js").expect("suffix disappeared"); - } - ign.tests.insert(test.to_owned().into_boxed_str()); - } - ign - }) - } else { - Ignored::default() - } -}); - /// Boa test262 tester #[derive(Debug, Parser)] #[command(author, version, about, name = "Boa test262 tester")] @@ -217,6 +163,10 @@ enum Cli { /// Execute tests serially #[arg(short, long)] disable_parallelism: bool, + + /// Path to a TOML file with the ignored tests, features, flags and/or files. + #[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)] + ignored: PathBuf, }, /// Compare two test suite results. Compare { @@ -235,7 +185,8 @@ enum Cli { } /// Program entry point. -fn main() { +fn main() -> Result<()> { + color_eyre::install()?; match Cli::parse() { Cli::Run { verbose, @@ -243,23 +194,15 @@ fn main() { suite, output, disable_parallelism, - } => { - if let Err(e) = run_test_suite( - verbose, - !disable_parallelism, - test262_path.as_path(), - suite.as_path(), - output.as_deref(), - ) { - eprintln!("Error: {e}"); - let mut src = e.source(); - while let Some(e) = src { - eprintln!(" caused by: {e}"); - src = e.source(); - } - std::process::exit(1); - } - } + ignored: ignore, + } => run_test_suite( + verbose, + !disable_parallelism, + test262_path.as_path(), + suite.as_path(), + output.as_deref(), + ignore.as_path(), + ), Cli::Compare { base, new, @@ -275,24 +218,33 @@ fn run_test_suite( test262_path: &Path, suite: &Path, output: Option<&Path>, -) -> anyhow::Result<()> { + ignored: &Path, +) -> Result<()> { if let Some(path) = output { if path.exists() { if !path.is_dir() { bail!("the output path must be a directory."); } } else { - fs::create_dir_all(path).context("could not create the output directory")?; + fs::create_dir_all(path).wrap_err("could not create the output directory")?; } } + let ignored = { + let mut input = String::new(); + let mut f = File::open(ignored).wrap_err("could not open ignored tests file")?; + f.read_to_string(&mut input) + .wrap_err("could not read ignored tests file")?; + toml::from_str(&input).wrap_err("could not decode ignored tests file")? + }; + if verbose != 0 { println!("Loading the test suite..."); } - let harness = read_harness(test262_path).context("could not read harness")?; + let harness = read_harness(test262_path).wrap_err("could not read harness")?; if suite.to_string_lossy().ends_with(".js") { - let test = read_test(&test262_path.join(suite)).with_context(|| { + let test = read_test(&test262_path.join(suite)).wrap_err_with(|| { let suite = suite.display(); format!("could not read the test {suite}") })?; @@ -304,7 +256,7 @@ fn run_test_suite( println!(); } else { - let suite = read_suite(&test262_path.join(suite)).with_context(|| { + let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| { let suite = suite.display(); format!("could not read the suite {suite}") })?; @@ -332,7 +284,7 @@ fn run_test_suite( ); write_json(results, output, verbose) - .context("could not write the results to the output JSON file")?; + .wrap_err("could not write the results to the output JSON file")?; } Ok(()) @@ -419,6 +371,7 @@ struct Test { includes: Box<[Box]>, locale: Locale, content: Box, + ignored: bool, } impl Test { @@ -440,15 +393,12 @@ impl Test { includes: metadata.includes, locale: metadata.locale, content: content.into(), + ignored: false, } } - /// Sets the name of the test. - fn set_name(&mut self, name: N) - where - N: Into>, - { - self.name = name.into(); + fn set_ignored(&mut self) { + self.ignored = true; } } @@ -534,6 +484,59 @@ where } } +impl<'de> Deserialize<'de> for TestFlags { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct FlagsVisitor; + + impl<'de> Visitor<'de> for FlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a sequence of flags") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut flags = TestFlags::empty(); + while let Some(elem) = seq.next_element::()? { + flags |= elem.into(); + } + Ok(flags) + } + } + + struct RawFlagsVisitor; + + impl<'de> Visitor<'de> for RawFlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a flags number") + } + + fn visit_u16(self, v: u16) -> Result + where + E: serde::de::Error, + { + TestFlags::from_bits(v).ok_or_else(|| { + E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") + }) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_seq(FlagsVisitor) + } else { + deserializer.deserialize_u16(RawFlagsVisitor) + } + } +} + /// Phase for an error. #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "lowercase")] diff --git a/boa_tester/src/read.rs b/boa_tester/src/read.rs index e198b64220..29dc714d63 100644 --- a/boa_tester/src/read.rs +++ b/boa_tester/src/read.rs @@ -1,10 +1,15 @@ //! Module to read the list of test suites from disk. -use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED}; -use anyhow::Context; +use crate::Ignored; + +use super::{Harness, Locale, Phase, Test, TestSuite}; +use color_eyre::{ + eyre::{eyre, WrapErr}, + Result, +}; use fxhash::FxHashMap; use serde::Deserialize; -use std::{fs, io, path::Path, str::FromStr}; +use std::{fs, io, path::Path}; /// Representation of the YAML metadata in Test262 tests. #[derive(Debug, Clone, Deserialize)] @@ -77,31 +82,12 @@ pub(super) enum TestFlag { NonDeterministic, } -impl FromStr for TestFlag { - type Err = String; // TODO: improve error type. - - fn from_str(s: &str) -> Result { - match s { - "onlyStrict" => Ok(Self::OnlyStrict), - "noStrict" => Ok(Self::NoStrict), - "module" => Ok(Self::Module), - "raw" => Ok(Self::Raw), - "async" => Ok(Self::Async), - "generated" => Ok(Self::Generated), - "CanBlockIsFalse" => Ok(Self::CanBlockIsFalse), - "CanBlockIsTrue" => Ok(Self::CanBlockIsTrue), - "non-deterministic" => Ok(Self::NonDeterministic), - _ => Err(format!("unknown test flag: {s}")), - } - } -} - /// Reads the Test262 defined bindings. -pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result { +pub(super) fn read_harness(test262_path: &Path) -> Result { let mut includes = FxHashMap::default(); - for entry in - fs::read_dir(test262_path.join("harness")).context("error reading the harness directory")? + for entry in fs::read_dir(test262_path.join("harness")) + .wrap_err("error reading the harness directory")? { let entry = entry?; let file_name = entry.file_name(); @@ -112,7 +98,7 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result { } let content = fs::read_to_string(entry.path()) - .with_context(|| format!("error reading the harnes/{file_name}"))?; + .wrap_err_with(|| format!("error reading the harnes/{file_name}"))?; includes.insert( file_name.into_owned().into_boxed_str(), @@ -120,13 +106,13 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result { ); } let assert = fs::read_to_string(test262_path.join("harness/assert.js")) - .context("error reading harnes/assert.js")? + .wrap_err("error reading harnes/assert.js")? .into_boxed_str(); let sta = fs::read_to_string(test262_path.join("harness/sta.js")) - .context("error reading harnes/sta.js")? + .wrap_err("error reading harnes/sta.js")? .into_boxed_str(); let doneprint_handle = fs::read_to_string(test262_path.join("harness/doneprintHandle.js")) - .context("error reading harnes/doneprintHandle.js")? + .wrap_err("error reading harnes/doneprintHandle.js")? .into_boxed_str(); Ok(Harness { @@ -138,38 +124,54 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result { } /// Reads a test suite in the given path. -pub(super) fn read_suite(path: &Path) -> anyhow::Result { +pub(super) fn read_suite( + path: &Path, + ignored: &Ignored, + mut ignore_suite: bool, +) -> Result { let name = path .file_name() - .with_context(|| format!("test suite with no name found: {}", path.display()))? + .ok_or_else(|| eyre!(format!("test suite with no name found: {}", path.display())))? .to_str() - .with_context(|| format!("non-UTF-8 suite name found: {}", path.display()))?; + .ok_or_else(|| eyre!(format!("non-UTF-8 suite name found: {}", path.display())))?; + + ignore_suite |= ignored.contains_test(name); let mut suites = Vec::new(); let mut tests = Vec::new(); // TODO: iterate in parallel - for entry in path.read_dir().context("retrieving entry")? { + for entry in path.read_dir().wrap_err("retrieving entry")? { let entry = entry?; - if entry.file_type().context("retrieving file type")?.is_dir() { - suites.push(read_suite(entry.path().as_path()).with_context(|| { - let path = entry.path(); - let suite = path.display(); - format!("error reading sub-suite {suite}") - })?); + if entry.file_type().wrap_err("retrieving file type")?.is_dir() { + suites.push( + read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| { + let path = entry.path(); + let suite = path.display(); + format!("error reading sub-suite {suite}") + })?, + ); } else if entry.file_name().to_string_lossy().contains("_FIXTURE") { continue; - } else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) { - let mut test = Test::default(); - test.set_name(entry.file_name().to_string_lossy()); - tests.push(test); } else { - tests.push(read_test(entry.path().as_path()).with_context(|| { + let mut test = read_test(entry.path().as_path()).wrap_err_with(|| { let path = entry.path(); let suite = path.display(); format!("error reading test {suite}") - })?); + })?; + + if ignore_suite + || ignored.contains_any_flag(test.flags) + || ignored.contains_test(&test.name) + || test + .features + .iter() + .any(|feat| ignored.contains_feature(feat)) + { + test.set_ignored(); + } + tests.push(test); } } diff --git a/boa_tester/src/results.rs b/boa_tester/src/results.rs index c8063d28d1..f1edfa7ebc 100644 --- a/boa_tester/src/results.rs +++ b/boa_tester/src/results.rs @@ -1,4 +1,5 @@ use super::SuiteResult; +use color_eyre::{eyre::WrapErr, Result}; use serde::{Deserialize, Serialize}; use std::{ env, fs, @@ -208,16 +209,16 @@ fn update_gh_pages_repo(path: &Path, verbose: u8) { } /// Compares the results of two test suite runs. -pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) { +pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result<()> { let base_results: ResultInfo = serde_json::from_reader(BufReader::new( - fs::File::open(base).expect("could not open the base results file"), + fs::File::open(base).wrap_err("could not open the base results file")?, )) - .expect("could not read the base results"); + .wrap_err("could not read the base results")?; let new_results: ResultInfo = serde_json::from_reader(BufReader::new( - fs::File::open(new).expect("could not open the new results file"), + fs::File::open(new).wrap_err("could not open the new results file")?, )) - .expect("could not read the new results"); + .wrap_err("could not read the new results")?; let base_total = base_results.results.total as isize; let new_total = new_results.results.total as isize; @@ -433,6 +434,8 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) { } } } + + Ok(()) } /// Test differences. diff --git a/test_ignore.toml b/test_ignore.toml new file mode 100644 index 0000000000..d760c8772c --- /dev/null +++ b/test_ignore.toml @@ -0,0 +1,45 @@ +# Not implemented yet: +flags = ["module"] + +features = [ + # Non-implemented features: + "json-modules", + "SharedArrayBuffer", + "resizable-arraybuffer", + "Temporal", + "tail-call-optimization", + "ShadowRealm", + "FinalizationRegistry", + "Atomics", + "dynamic_import", + "decorators", + "array-grouping", + "IsHTMLDDA", + + # Non-implemented Intl features + "intl-normative-optional", + "Intl.DurationFormat", + "Intl.NumberFormat-v3", + "Intl.NumberFormat-unified", + "Intl.ListFormat", + "Intl.DisplayNames", + "Intl.RelativeTimeFormat", + "Intl.Segmenter", + "Intl.Locale", + + # Stage 3 proposals + + # https://github.com/tc39/proposal-symbols-as-weakmap-keys + "symbols-as-weakmap-keys", + + # Non-standard + "caller", + + # RegExp tests that check individual codepoints. + # They are not useful considering the cpu time they waste. + "regexp-unicode-property-escapes", +] + +# RegExp tests that check individual codepoints. +# They are not useful considering the cpu time they waste. +tests = ["CharacterClassEscapes"] diff --git a/test_ignore.txt b/test_ignore.txt deleted file mode 100644 index 4c1e07f946..0000000000 --- a/test_ignore.txt +++ /dev/null @@ -1,68 +0,0 @@ -// Not implemented yet: -flag:module - -// Non-implemented features: -feature:json-modules -feature:SharedArrayBuffer -feature:resizable-arraybuffer -feature:Temporal -feature:tail-call-optimization -feature:ShadowRealm -feature:FinalizationRegistry -feature:FinalizationRegistry.prototype.cleanupSome -feature:Atomics -feature:dynamic_import -feature:decorators -feature:array-grouping -feature:IsHTMLDDA - -// Non-implemented Intl features -feature:intl-normative-optional -feature:Intl.DurationFormat -feature:Intl.NumberFormat-v3 -feature:Intl.NumberFormat-unified -feature:Intl.ListFormat -feature:Intl.DisplayNames -feature:Intl.RelativeTimeFormat -feature:Intl.Segmenter -feature:Intl.Locale - -// Non-standard -feature:caller - -// Stage 3 proposals - -// https://github.com/tc39/proposal-symbols-as-weakmap-keys -feature:symbols-as-weakmap-keys - -// These generate a stack overflow -tco-call -tco-member - -// RegExp tests that check individual codepoints. -// They are not usefull in comparision to the cpu time they waste. -feature:regexp-unicode-property-escapes -character-class-non-whitespace-class-escape-plus-quantifier -character-class-non-whitespace-class-escape-plus-quantifier-flags-u -character-class-non-word-class-escape-flags-u -character-class-non-whitespace-class-escape -character-class-non-digit-class-escape-plus-quantifier-flags-u -character-class-non-word-class-escape -character-class-non-whitespace-class-escape-flags-u -character-class-digit-class-escape-flags-u -character-class-digit-class-escape -character-class-non-digit-class-escape-plus-quantifier -character-class-whitespace-class-escape-flags-u -character-class-whitespace-class-escape-plus-quantifier-flags-u -character-class-word-class-escape -character-class-digit-class-escape-plus-quantifier-flags-u -character-class-non-digit-class-escape -character-class-digit-class-escape-plus-quantifier -character-class-whitespace-class-escape -character-class-whitespace-class-escape-plus-quantifier -character-class-word-class-escape-plus-quantifier -character-class-non-word-class-escape-plus-quantifier -character-class-word-class-escape-flags-u -character-class-word-class-escape-plus-quantifier-flags-u -character-class-non-digit-class-escape-flags-u -character-class-non-word-class-escape-plus-quantifier-flags-u