Browse Source

Cleanup `boa_tester` (#2440)

Just some quality changes to improve the maintainability of the tester:

- Replaces `anyhow` with `color_eyre` to have a better output on errors/panics.
- Changes the ignore file to a TOML file and replaces all parsing logic with the `toml` crate.
- Adds a `ignored` field on all `Test`s to simplify run logic.
- Replaces the global `IGNORED` with an `ignored` argument on the CLI.
pull/2443/head
José Julián Espina 2 years ago
parent
commit
9c7b4d5531
  1. 199
      Cargo.lock
  2. 3
      boa_tester/Cargo.toml
  3. 307
      boa_tester/src/exec/mod.rs
  4. 223
      boa_tester/src/main.rs
  5. 92
      boa_tester/src/read.rs
  6. 13
      boa_tester/src/results.rs
  7. 45
      test_ignore.toml
  8. 68
      test_ignore.txt

199
Cargo.lock generated

@ -2,6 +2,21 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "addr2line"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "0.7.19"
@ -26,12 +41,6 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anyhow"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
[[package]]
name = "arbitrary"
version = "1.2.0"
@ -58,6 +67,21 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backtrace"
version = "0.3.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -210,13 +234,13 @@ dependencies = [
name = "boa_tester"
version = "0.16.0"
dependencies = [
"anyhow",
"bitflags",
"boa_engine",
"boa_gc",
"boa_interner",
"boa_parser",
"clap 4.0.25",
"color-eyre",
"colored",
"fxhash",
"once_cell",
@ -225,6 +249,7 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
"toml",
]
[[package]]
@ -394,6 +419,33 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "color-eyre"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204"
dependencies = [
"backtrace",
"color-spantrace",
"eyre",
"indenter",
"once_cell",
"owo-colors",
"tracing-error",
]
[[package]]
name = "color-spantrace"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
dependencies = [
"once_cell",
"owo-colors",
"tracing-core",
"tracing-error",
]
[[package]]
name = "colored"
version = "2.0.0"
@ -626,6 +678,16 @@ dependencies = [
"str-buf",
]
[[package]]
name = "eyre"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "fast-float"
version = "0.2.0"
@ -692,6 +754,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "gimli"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
[[package]]
name = "half"
version = "1.8.2"
@ -872,6 +940,12 @@ dependencies = [
"icu_provider_blob",
]
[[package]]
name = "indenter"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
version = "1.9.1"
@ -1036,6 +1110,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
dependencies = [
"adler",
]
[[package]]
name = "nibble_vec"
version = "0.1.0"
@ -1098,6 +1181,15 @@ dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.16.0"
@ -1116,6 +1208,12 @@ version = "6.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9"
[[package]]
name = "owo-colors"
version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1192,6 +1290,12 @@ dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "plotters"
version = "0.3.4"
@ -1394,6 +1498,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -1525,6 +1635,15 @@ dependencies = [
"unsafe-libyaml",
]
[[package]]
name = "sharded-slab"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
dependencies = [
"lazy_static",
]
[[package]]
name = "siphasher"
version = "0.3.10"
@ -1647,6 +1766,15 @@ dependencies = [
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.1.44"
@ -1695,6 +1823,57 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "toml"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
dependencies = [
"serde",
]
[[package]]
name = "tracing"
version = "0.1.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if",
"pin-project-lite",
"tracing-core",
]
[[package]]
name = "tracing-core"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-error"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
dependencies = [
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
dependencies = [
"sharded-slab",
"thread_local",
"tracing-core",
]
[[package]]
name = "unicode-general-category"
version = "0.6.0"
@ -1746,6 +1925,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "version_check"
version = "0.9.4"

3
boa_tester/Cargo.toml

@ -26,4 +26,5 @@ once_cell = "1.16.0"
colored = "2.0.0"
fxhash = "0.2.1"
rayon = "1.5.3"
anyhow = "1.0.66"
toml = "0.5.9"
color-eyre = "0.6.2"

307
boa_tester/src/exec/mod.rs

@ -2,11 +2,12 @@
mod js262;
use std::borrow::Cow;
use crate::read::ErrorType;
use super::{
Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult,
TestSuite, IGNORED,
Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, TestSuite,
};
use boa_engine::{
builtins::JsArgs, object::FunctionBuilder, property::Attribute, Context, JsNativeErrorKind,
@ -130,6 +131,24 @@ impl Test {
/// Runs the test once, in strict or non-strict mode
fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
if self.ignored {
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
"Ignored".yellow()
);
} else {
print!("{}", "-".yellow());
}
return TestResult {
name: self.name.clone(),
strict,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
};
}
if verbose > 1 {
println!(
"`{}`{}: starting",
@ -139,183 +158,143 @@ impl Test {
}
let test_content = if strict {
format!("\"use strict\";\n{}", self.content)
Cow::Owned(format!("\"use strict\";\n{}", self.content))
} else {
self.content.to_string()
Cow::Borrowed(&*self.content)
};
let (result, result_text) = if !IGNORED.contains_any_flag(self.flags)
&& !IGNORED.contains_test(&self.name)
&& !IGNORED.contains_any_feature(&self.features)
&& (matches!(self.expected_outcome, Outcome::Positive)
|| matches!(
self.expected_outcome,
Outcome::Negative {
phase: Phase::Parse,
error_type: _,
}
)
|| matches!(
self.expected_outcome,
Outcome::Negative {
phase: Phase::Early,
error_type: _,
}
)
|| matches!(
self.expected_outcome,
Outcome::Negative {
phase: Phase::Runtime,
error_type: _,
}
)) {
let res = std::panic::catch_unwind(|| match self.expected_outcome {
Outcome::Positive => {
let mut context = Context::default();
let async_result = AsyncResult::default();
if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) {
return (false, e);
}
let result = std::panic::catch_unwind(|| match self.expected_outcome {
Outcome::Positive => {
let mut context = Context::default();
let async_result = AsyncResult::default();
// TODO: timeout
let value = match context.eval(&test_content) {
Ok(v) => v,
Err(e) => return (false, format!("Uncaught {e}")),
};
if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) {
return (false, e);
}
if let Err(e) = async_result.inner.borrow().as_ref() {
return (false, format!("Uncaught {e}"));
}
// TODO: timeout
let value = match context.eval(&*test_content) {
Ok(v) => v,
Err(e) => return (false, format!("Uncaught {e}")),
};
(true, value.display().to_string())
if let Err(e) = async_result.inner.borrow().as_ref() {
return (false, format!("Uncaught {e}"));
}
Outcome::Negative {
phase: Phase::Parse | Phase::Early,
(true, value.display().to_string())
}
Outcome::Negative {
phase: Phase::Parse | Phase::Early,
error_type,
} => {
assert_eq!(
error_type,
} => {
assert_eq!(
error_type,
ErrorType::SyntaxError,
"non-SyntaxError parsing/early error found in {}",
self.name
);
let mut context = Context::default();
match context.parse(&test_content) {
Ok(statement_list) => match context.compile(&statement_list) {
Ok(_) => (false, "StatementList compilation should fail".to_owned()),
Err(e) => (true, format!("Uncaught {e:?}")),
},
Err(e) => (true, format!("Uncaught {e}")),
}
ErrorType::SyntaxError,
"non-SyntaxError parsing/early error found in {}",
self.name
);
let mut context = Context::default();
match context.parse(&*test_content) {
Ok(statement_list) => match context.compile(&statement_list) {
Ok(_) => (false, "StatementList compilation should fail".to_owned()),
Err(e) => (true, format!("Uncaught {e:?}")),
},
Err(e) => (true, format!("Uncaught {e}")),
}
Outcome::Negative {
phase: Phase::Resolution,
error_type: _,
} => todo!("check module resolution errors"),
Outcome::Negative {
phase: Phase::Runtime,
error_type,
} => {
let mut context = Context::default();
if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) {
return (false, e);
}
let code = match Parser::new(test_content.as_bytes())
.parse_all(context.interner_mut())
.map_err(Into::into)
.and_then(|stmts| context.compile(&stmts))
{
Ok(code) => code,
Err(e) => return (false, format!("Uncaught {e}")),
};
// TODO: timeout
let e = match context.execute(code) {
Ok(res) => return (false, res.display().to_string()),
Err(e) => e,
};
if let Ok(e) = e.try_native(&mut context) {
match &e.kind {
JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
JsNativeErrorKind::Reference
if error_type == ErrorType::ReferenceError => {}
JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
_ => return (false, format!("Uncaught {e}")),
}
Outcome::Negative {
phase: Phase::Resolution,
error_type: _,
} => todo!("check module resolution errors"),
Outcome::Negative {
phase: Phase::Runtime,
error_type,
} => {
let mut context = Context::default();
if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) {
return (false, e);
}
let code = match Parser::new(test_content.as_bytes())
.parse_all(context.interner_mut())
.map_err(Into::into)
.and_then(|stmts| context.compile(&stmts))
{
Ok(code) => code,
Err(e) => return (false, format!("Uncaught {e}")),
};
// TODO: timeout
let e = match context.execute(code) {
Ok(res) => return (false, res.display().to_string()),
Err(e) => e,
};
if let Ok(e) = e.try_native(&mut context) {
match &e.kind {
JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
JsNativeErrorKind::Reference if error_type == ErrorType::ReferenceError => {
}
(true, format!("Uncaught {e}"))
} else {
let passed = e
.as_opaque()
.expect("try_native cannot fail if e is not opaque")
.as_object()
.and_then(|o| o.get("constructor", &mut context).ok())
.as_ref()
.and_then(JsValue::as_object)
.and_then(|o| o.get("name", &mut context).ok())
.as_ref()
.and_then(JsValue::as_string)
.map(|s| s == error_type.as_str())
.unwrap_or_default();
(passed, format!("Uncaught {e}"))
JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
_ => return (false, format!("Uncaught {e}")),
}
(true, format!("Uncaught {e}"))
} else {
let passed = e
.as_opaque()
.expect("try_native cannot fail if e is not opaque")
.as_object()
.and_then(|o| o.get("constructor", &mut context).ok())
.as_ref()
.and_then(JsValue::as_object)
.and_then(|o| o.get("name", &mut context).ok())
.as_ref()
.and_then(JsValue::as_string)
.map(|s| s == error_type.as_str())
.unwrap_or_default();
(passed, format!("Uncaught {e}"))
}
});
let result = res.map_or_else(
|_| {
eprintln!("last panic was on test \"{}\"", self.name);
(TestOutcomeResult::Panic, String::new())
},
|(res, text)| {
if res {
(TestOutcomeResult::Passed, text)
} else {
(TestOutcomeResult::Failed, text)
}
},
);
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
if matches!(result, (TestOutcomeResult::Passed, _)) {
"Passed".green()
} else if matches!(result, (TestOutcomeResult::Failed, _)) {
"Failed".red()
} else {
"⚠ Panic ⚠".red()
}
);
} else {
print!(
"{}",
if matches!(result, (TestOutcomeResult::Passed, _)) {
".".green()
} else {
"F".red()
}
);
}
});
let (result, result_text) = result.map_or_else(
|_| {
eprintln!("last panic was on test \"{}\"", self.name);
(TestOutcomeResult::Panic, String::new())
},
|(res, text)| {
if res {
(TestOutcomeResult::Passed, text)
} else {
(TestOutcomeResult::Failed, text)
}
},
);
result
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
if result == TestOutcomeResult::Passed {
"Passed".green()
} else if result == TestOutcomeResult::Failed {
"Failed".red()
} else {
"⚠ Panic ⚠".red()
}
);
} else {
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
"Ignored".yellow()
);
} else {
print!("{}", "-".yellow());
}
(TestOutcomeResult::Ignored, String::new())
};
print!(
"{}",
if result == TestOutcomeResult::Passed {
".".green()
} else {
"F".red()
}
);
}
if verbose > 2 {
println!(

223
boa_tester/src/main.rs

@ -1,7 +1,7 @@
//! Test262 test runner
//!
//! This crate will run the full ECMAScript test suite (Test262) and report compliance of the
//! `boa` context.
//! `boa` wrap_err.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg",
html_favicon_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg"
@ -70,26 +70,34 @@ use self::{
read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
results::{compare_results, write_json},
};
use anyhow::{bail, Context};
use bitflags::bitflags;
use clap::{ArgAction, Parser, ValueHint};
use color_eyre::{
eyre::{bail, WrapErr},
Result,
};
use colored::Colorize;
use fxhash::{FxHashMap, FxHashSet};
use once_cell::sync::Lazy;
use read::ErrorType;
use serde::{Deserialize, Serialize};
use serde::{
de::{Unexpected, Visitor},
Deserialize, Deserializer, Serialize,
};
use std::{
fs,
fs::{self, File},
io::Read,
path::{Path, PathBuf},
};
/// Structure to allow defining ignored tests, features and files that should
/// be ignored even when reading.
#[derive(Debug)]
#[derive(Debug, Deserialize)]
struct Ignored {
#[serde(default)]
tests: FxHashSet<Box<str>>,
#[serde(default)]
features: FxHashSet<Box<str>>,
files: FxHashSet<Box<str>>,
#[serde(default = "TestFlags::empty")]
flags: TestFlags,
}
@ -102,16 +110,17 @@ impl Ignored {
/// Checks if the ignore list contains the given feature name in the list
/// of features to ignore.
pub(crate) fn contains_any_feature(&self, features: &[Box<str>]) -> bool {
features
.iter()
.any(|feature| self.features.contains(feature))
}
/// Checks if the ignore list contains the given file name in the list to
/// ignore from reading.
pub(crate) fn contains_file(&self, file: &str) -> bool {
self.files.contains(file)
pub(crate) fn contains_feature(&self, feature: &str) -> bool {
if self.features.contains(feature) {
return true;
}
// Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`.
// This ensures those are also ignored.
feature
.split('.')
.next()
.map(|feat| self.features.contains(feat))
.unwrap_or_default()
}
pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool {
@ -124,74 +133,11 @@ impl Default for Ignored {
Self {
tests: FxHashSet::default(),
features: FxHashSet::default(),
files: FxHashSet::default(),
flags: TestFlags::empty(),
}
}
}
/// List of ignored tests.
static IGNORED: Lazy<Ignored> = Lazy::new(|| {
let path = Path::new("test_ignore.txt");
if path.exists() {
let filtered = fs::read_to_string(path).expect("could not read test filters");
filtered
.lines()
.filter(|line| !line.is_empty() && !line.starts_with("//"))
.fold(Ignored::default(), |mut ign, line| {
// let mut line = line.to_owned();
if line.starts_with("file:") {
let file = line
.strip_prefix("file:")
.expect("prefix disappeared")
.trim()
.to_owned()
.into_boxed_str();
let test = if file.ends_with(".js") {
file.strip_suffix(".js")
.expect("suffix disappeared")
.to_owned()
.into_boxed_str()
} else {
file.clone()
};
ign.files.insert(file);
ign.tests.insert(test);
} else if line.starts_with("feature:") {
ign.features.insert(
line.strip_prefix("feature:")
.expect("prefix disappeared")
.trim()
.to_owned()
.into_boxed_str(),
);
} else if line.starts_with("flag:") {
let flag = line
.strip_prefix("flag:")
.expect("prefix disappeared")
.trim()
.parse::<TestFlag>()
.expect("invalid flag found");
ign.flags.insert(flag.into());
} else {
let mut test = line.trim();
if test
.rsplit('.')
.next()
.map(|ext| ext.eq_ignore_ascii_case("js"))
== Some(true)
{
test = test.strip_suffix(".js").expect("suffix disappeared");
}
ign.tests.insert(test.to_owned().into_boxed_str());
}
ign
})
} else {
Ignored::default()
}
});
/// Boa test262 tester
#[derive(Debug, Parser)]
#[command(author, version, about, name = "Boa test262 tester")]
@ -217,6 +163,10 @@ enum Cli {
/// Execute tests serially
#[arg(short, long)]
disable_parallelism: bool,
/// Path to a TOML file with the ignored tests, features, flags and/or files.
#[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)]
ignored: PathBuf,
},
/// Compare two test suite results.
Compare {
@ -235,7 +185,8 @@ enum Cli {
}
/// Program entry point.
fn main() {
fn main() -> Result<()> {
color_eyre::install()?;
match Cli::parse() {
Cli::Run {
verbose,
@ -243,23 +194,15 @@ fn main() {
suite,
output,
disable_parallelism,
} => {
if let Err(e) = run_test_suite(
verbose,
!disable_parallelism,
test262_path.as_path(),
suite.as_path(),
output.as_deref(),
) {
eprintln!("Error: {e}");
let mut src = e.source();
while let Some(e) = src {
eprintln!(" caused by: {e}");
src = e.source();
}
std::process::exit(1);
}
}
ignored: ignore,
} => run_test_suite(
verbose,
!disable_parallelism,
test262_path.as_path(),
suite.as_path(),
output.as_deref(),
ignore.as_path(),
),
Cli::Compare {
base,
new,
@ -275,24 +218,33 @@ fn run_test_suite(
test262_path: &Path,
suite: &Path,
output: Option<&Path>,
) -> anyhow::Result<()> {
ignored: &Path,
) -> Result<()> {
if let Some(path) = output {
if path.exists() {
if !path.is_dir() {
bail!("the output path must be a directory.");
}
} else {
fs::create_dir_all(path).context("could not create the output directory")?;
fs::create_dir_all(path).wrap_err("could not create the output directory")?;
}
}
let ignored = {
let mut input = String::new();
let mut f = File::open(ignored).wrap_err("could not open ignored tests file")?;
f.read_to_string(&mut input)
.wrap_err("could not read ignored tests file")?;
toml::from_str(&input).wrap_err("could not decode ignored tests file")?
};
if verbose != 0 {
println!("Loading the test suite...");
}
let harness = read_harness(test262_path).context("could not read harness")?;
let harness = read_harness(test262_path).wrap_err("could not read harness")?;
if suite.to_string_lossy().ends_with(".js") {
let test = read_test(&test262_path.join(suite)).with_context(|| {
let test = read_test(&test262_path.join(suite)).wrap_err_with(|| {
let suite = suite.display();
format!("could not read the test {suite}")
})?;
@ -304,7 +256,7 @@ fn run_test_suite(
println!();
} else {
let suite = read_suite(&test262_path.join(suite)).with_context(|| {
let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| {
let suite = suite.display();
format!("could not read the suite {suite}")
})?;
@ -332,7 +284,7 @@ fn run_test_suite(
);
write_json(results, output, verbose)
.context("could not write the results to the output JSON file")?;
.wrap_err("could not write the results to the output JSON file")?;
}
Ok(())
@ -419,6 +371,7 @@ struct Test {
includes: Box<[Box<str>]>,
locale: Locale,
content: Box<str>,
ignored: bool,
}
impl Test {
@ -440,15 +393,12 @@ impl Test {
includes: metadata.includes,
locale: metadata.locale,
content: content.into(),
ignored: false,
}
}
/// Sets the name of the test.
fn set_name<N>(&mut self, name: N)
where
N: Into<Box<str>>,
{
self.name = name.into();
fn set_ignored(&mut self) {
self.ignored = true;
}
}
@ -534,6 +484,59 @@ where
}
}
impl<'de> Deserialize<'de> for TestFlags {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct FlagsVisitor;
impl<'de> Visitor<'de> for FlagsVisitor {
type Value = TestFlags;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "a sequence of flags")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'de>,
{
let mut flags = TestFlags::empty();
while let Some(elem) = seq.next_element::<TestFlag>()? {
flags |= elem.into();
}
Ok(flags)
}
}
struct RawFlagsVisitor;
impl<'de> Visitor<'de> for RawFlagsVisitor {
type Value = TestFlags;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "a flags number")
}
fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
TestFlags::from_bits(v).ok_or_else(|| {
E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number")
})
}
}
if deserializer.is_human_readable() {
deserializer.deserialize_seq(FlagsVisitor)
} else {
deserializer.deserialize_u16(RawFlagsVisitor)
}
}
}
/// Phase for an error.
#[derive(Debug, Clone, Copy, Deserialize)]
#[serde(rename_all = "lowercase")]

92
boa_tester/src/read.rs

@ -1,10 +1,15 @@
//! Module to read the list of test suites from disk.
use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED};
use anyhow::Context;
use crate::Ignored;
use super::{Harness, Locale, Phase, Test, TestSuite};
use color_eyre::{
eyre::{eyre, WrapErr},
Result,
};
use fxhash::FxHashMap;
use serde::Deserialize;
use std::{fs, io, path::Path, str::FromStr};
use std::{fs, io, path::Path};
/// Representation of the YAML metadata in Test262 tests.
#[derive(Debug, Clone, Deserialize)]
@ -77,31 +82,12 @@ pub(super) enum TestFlag {
NonDeterministic,
}
impl FromStr for TestFlag {
type Err = String; // TODO: improve error type.
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"onlyStrict" => Ok(Self::OnlyStrict),
"noStrict" => Ok(Self::NoStrict),
"module" => Ok(Self::Module),
"raw" => Ok(Self::Raw),
"async" => Ok(Self::Async),
"generated" => Ok(Self::Generated),
"CanBlockIsFalse" => Ok(Self::CanBlockIsFalse),
"CanBlockIsTrue" => Ok(Self::CanBlockIsTrue),
"non-deterministic" => Ok(Self::NonDeterministic),
_ => Err(format!("unknown test flag: {s}")),
}
}
}
/// Reads the Test262 defined bindings.
pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
pub(super) fn read_harness(test262_path: &Path) -> Result<Harness> {
let mut includes = FxHashMap::default();
for entry in
fs::read_dir(test262_path.join("harness")).context("error reading the harness directory")?
for entry in fs::read_dir(test262_path.join("harness"))
.wrap_err("error reading the harness directory")?
{
let entry = entry?;
let file_name = entry.file_name();
@ -112,7 +98,7 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
}
let content = fs::read_to_string(entry.path())
.with_context(|| format!("error reading the harnes/{file_name}"))?;
.wrap_err_with(|| format!("error reading the harnes/{file_name}"))?;
includes.insert(
file_name.into_owned().into_boxed_str(),
@ -120,13 +106,13 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
);
}
let assert = fs::read_to_string(test262_path.join("harness/assert.js"))
.context("error reading harnes/assert.js")?
.wrap_err("error reading harnes/assert.js")?
.into_boxed_str();
let sta = fs::read_to_string(test262_path.join("harness/sta.js"))
.context("error reading harnes/sta.js")?
.wrap_err("error reading harnes/sta.js")?
.into_boxed_str();
let doneprint_handle = fs::read_to_string(test262_path.join("harness/doneprintHandle.js"))
.context("error reading harnes/doneprintHandle.js")?
.wrap_err("error reading harnes/doneprintHandle.js")?
.into_boxed_str();
Ok(Harness {
@ -138,38 +124,54 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
}
/// Reads a test suite in the given path.
pub(super) fn read_suite(path: &Path) -> anyhow::Result<TestSuite> {
pub(super) fn read_suite(
path: &Path,
ignored: &Ignored,
mut ignore_suite: bool,
) -> Result<TestSuite> {
let name = path
.file_name()
.with_context(|| format!("test suite with no name found: {}", path.display()))?
.ok_or_else(|| eyre!(format!("test suite with no name found: {}", path.display())))?
.to_str()
.with_context(|| format!("non-UTF-8 suite name found: {}", path.display()))?;
.ok_or_else(|| eyre!(format!("non-UTF-8 suite name found: {}", path.display())))?;
ignore_suite |= ignored.contains_test(name);
let mut suites = Vec::new();
let mut tests = Vec::new();
// TODO: iterate in parallel
for entry in path.read_dir().context("retrieving entry")? {
for entry in path.read_dir().wrap_err("retrieving entry")? {
let entry = entry?;
if entry.file_type().context("retrieving file type")?.is_dir() {
suites.push(read_suite(entry.path().as_path()).with_context(|| {
let path = entry.path();
let suite = path.display();
format!("error reading sub-suite {suite}")
})?);
if entry.file_type().wrap_err("retrieving file type")?.is_dir() {
suites.push(
read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| {
let path = entry.path();
let suite = path.display();
format!("error reading sub-suite {suite}")
})?,
);
} else if entry.file_name().to_string_lossy().contains("_FIXTURE") {
continue;
} else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) {
let mut test = Test::default();
test.set_name(entry.file_name().to_string_lossy());
tests.push(test);
} else {
tests.push(read_test(entry.path().as_path()).with_context(|| {
let mut test = read_test(entry.path().as_path()).wrap_err_with(|| {
let path = entry.path();
let suite = path.display();
format!("error reading test {suite}")
})?);
})?;
if ignore_suite
|| ignored.contains_any_flag(test.flags)
|| ignored.contains_test(&test.name)
|| test
.features
.iter()
.any(|feat| ignored.contains_feature(feat))
{
test.set_ignored();
}
tests.push(test);
}
}

13
boa_tester/src/results.rs

@ -1,4 +1,5 @@
use super::SuiteResult;
use color_eyre::{eyre::WrapErr, Result};
use serde::{Deserialize, Serialize};
use std::{
env, fs,
@ -208,16 +209,16 @@ fn update_gh_pages_repo(path: &Path, verbose: u8) {
}
/// Compares the results of two test suite runs.
pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result<()> {
let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
fs::File::open(base).expect("could not open the base results file"),
fs::File::open(base).wrap_err("could not open the base results file")?,
))
.expect("could not read the base results");
.wrap_err("could not read the base results")?;
let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
fs::File::open(new).expect("could not open the new results file"),
fs::File::open(new).wrap_err("could not open the new results file")?,
))
.expect("could not read the new results");
.wrap_err("could not read the new results")?;
let base_total = base_results.results.total as isize;
let new_total = new_results.results.total as isize;
@ -433,6 +434,8 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
}
}
}
Ok(())
}
/// Test differences.

45
test_ignore.toml

@ -0,0 +1,45 @@
# Not implemented yet:
flags = ["module"]
features = [
# Non-implemented features:
"json-modules",
"SharedArrayBuffer",
"resizable-arraybuffer",
"Temporal",
"tail-call-optimization",
"ShadowRealm",
"FinalizationRegistry",
"Atomics",
"dynamic_import",
"decorators",
"array-grouping",
"IsHTMLDDA",
# Non-implemented Intl features
"intl-normative-optional",
"Intl.DurationFormat",
"Intl.NumberFormat-v3",
"Intl.NumberFormat-unified",
"Intl.ListFormat",
"Intl.DisplayNames",
"Intl.RelativeTimeFormat",
"Intl.Segmenter",
"Intl.Locale",
# Stage 3 proposals
# https://github.com/tc39/proposal-symbols-as-weakmap-keys
"symbols-as-weakmap-keys",
# Non-standard
"caller",
# RegExp tests that check individual codepoints.
# They are not useful considering the cpu time they waste.
"regexp-unicode-property-escapes",
]
# RegExp tests that check individual codepoints.
# They are not useful considering the cpu time they waste.
tests = ["CharacterClassEscapes"]

68
test_ignore.txt

@ -1,68 +0,0 @@
// Not implemented yet:
flag:module
// Non-implemented features:
feature:json-modules
feature:SharedArrayBuffer
feature:resizable-arraybuffer
feature:Temporal
feature:tail-call-optimization
feature:ShadowRealm
feature:FinalizationRegistry
feature:FinalizationRegistry.prototype.cleanupSome
feature:Atomics
feature:dynamic_import
feature:decorators
feature:array-grouping
feature:IsHTMLDDA
// Non-implemented Intl features
feature:intl-normative-optional
feature:Intl.DurationFormat
feature:Intl.NumberFormat-v3
feature:Intl.NumberFormat-unified
feature:Intl.ListFormat
feature:Intl.DisplayNames
feature:Intl.RelativeTimeFormat
feature:Intl.Segmenter
feature:Intl.Locale
// Non-standard
feature:caller
// Stage 3 proposals
// https://github.com/tc39/proposal-symbols-as-weakmap-keys
feature:symbols-as-weakmap-keys
// These generate a stack overflow
tco-call
tco-member
// RegExp tests that check individual codepoints.
// They are not usefull in comparision to the cpu time they waste.
feature:regexp-unicode-property-escapes
character-class-non-whitespace-class-escape-plus-quantifier
character-class-non-whitespace-class-escape-plus-quantifier-flags-u
character-class-non-word-class-escape-flags-u
character-class-non-whitespace-class-escape
character-class-non-digit-class-escape-plus-quantifier-flags-u
character-class-non-word-class-escape
character-class-non-whitespace-class-escape-flags-u
character-class-digit-class-escape-flags-u
character-class-digit-class-escape
character-class-non-digit-class-escape-plus-quantifier
character-class-whitespace-class-escape-flags-u
character-class-whitespace-class-escape-plus-quantifier-flags-u
character-class-word-class-escape
character-class-digit-class-escape-plus-quantifier-flags-u
character-class-non-digit-class-escape
character-class-digit-class-escape-plus-quantifier
character-class-whitespace-class-escape
character-class-whitespace-class-escape-plus-quantifier
character-class-word-class-escape-plus-quantifier
character-class-non-word-class-escape-plus-quantifier
character-class-word-class-escape-flags-u
character-class-word-class-escape-plus-quantifier-flags-u
character-class-non-digit-class-escape-flags-u
character-class-non-word-class-escape-plus-quantifier-flags-u
Loading…
Cancel
Save