github
/
boa
mirror of https://github.com/boa-dev/boa.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Cleanup `boa_tester` (#2440) 

Just some quality changes to improve the maintainability of the tester:

- Replaces `anyhow` with `color_eyre` to have a better output on errors/panics.
- Changes the ignore file to a TOML file and replaces all parsing logic with the `toml` crate.
- Adds a `ignored` field on all `Test`s to simplify run logic.
- Replaces the global `IGNORED` with an `ignored` argument on the CLI.
pull/2443/head
José Julián Espina 2 years ago
parent
ace098fcc5
commit
9c7b4d5531
8 changed files with 550 additions and 400 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 199
      Cargo.lock
  2. 3
      boa_tester/Cargo.toml
  3. 307
      boa_tester/src/exec/mod.rs
  4. 223
      boa_tester/src/main.rs
  5. 92
      boa_tester/src/read.rs
  6. 13
      boa_tester/src/results.rs
  7. 45
      test_ignore.toml
  8. 68
      test_ignore.txt

199
Cargo.lock generated
Unescape View File

@ -2,6 +2,21 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "addr2line"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "0.7.19"
@ -26,12 +41,6 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anyhow"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
[[package]]
name = "arbitrary"
version = "1.2.0"
@ -58,6 +67,21 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backtrace"
version = "0.3.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -210,13 +234,13 @@ dependencies = [
name = "boa_tester"
version = "0.16.0"
dependencies = [
"anyhow",
"bitflags",
"boa_engine",
"boa_gc",
"boa_interner",
"boa_parser",
"clap 4.0.25",
"color-eyre",
"colored",
"fxhash",
"once_cell",
@ -225,6 +249,7 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
"toml",
]
[[package]]
@ -394,6 +419,33 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "color-eyre"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204"
dependencies = [
"backtrace",
"color-spantrace",
"eyre",
"indenter",
"once_cell",
"owo-colors",
"tracing-error",
]
[[package]]
name = "color-spantrace"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
dependencies = [
"once_cell",
"owo-colors",
"tracing-core",
"tracing-error",
]
[[package]]
name = "colored"
version = "2.0.0"
@ -626,6 +678,16 @@ dependencies = [
"str-buf",
]
[[package]]
name = "eyre"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "fast-float"
version = "0.2.0"
@ -692,6 +754,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "gimli"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
[[package]]
name = "half"
version = "1.8.2"
@ -872,6 +940,12 @@ dependencies = [
"icu_provider_blob",
]
[[package]]
name = "indenter"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
version = "1.9.1"
@ -1036,6 +1110,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
dependencies = [
"adler",
]
[[package]]
name = "nibble_vec"
version = "0.1.0"
@ -1098,6 +1181,15 @@ dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.16.0"
@ -1116,6 +1208,12 @@ version = "6.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9"
[[package]]
name = "owo-colors"
version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1192,6 +1290,12 @@ dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "plotters"
version = "0.3.4"
@ -1394,6 +1498,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -1525,6 +1635,15 @@ dependencies = [
"unsafe-libyaml",
]
[[package]]
name = "sharded-slab"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
dependencies = [
"lazy_static",
]
[[package]]
name = "siphasher"
version = "0.3.10"
@ -1647,6 +1766,15 @@ dependencies = [
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.1.44"
@ -1695,6 +1823,57 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "toml"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
dependencies = [
"serde",
]
[[package]]
name = "tracing"
version = "0.1.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if",
"pin-project-lite",
"tracing-core",
]
[[package]]
name = "tracing-core"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-error"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
dependencies = [
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
dependencies = [
"sharded-slab",
"thread_local",
"tracing-core",
]
[[package]]
name = "unicode-general-category"
version = "0.6.0"
@ -1746,6 +1925,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "version_check"
version = "0.9.4"

3
boa_tester/Cargo.toml
Unescape View File

@ -26,4 +26,5 @@ once_cell = "1.16.0"
colored = "2.0.0"
fxhash = "0.2.1"
rayon = "1.5.3"
anyhow = "1.0.66"
toml = "0.5.9"
color-eyre = "0.6.2"

307
boa_tester/src/exec/mod.rs
Unescape View File

@ -2,11 +2,12 @@
mod js262;
use std::borrow::Cow;
use crate::read::ErrorType;
use super::{
Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult,
TestSuite, IGNORED,
Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, TestSuite,
};
use boa_engine::{
builtins::JsArgs, object::FunctionBuilder, property::Attribute, Context, JsNativeErrorKind,
@ -130,6 +131,24 @@ impl Test {
/// Runs the test once, in strict or non-strict mode
fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
if self.ignored {
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
"Ignored".yellow()
);
} else {
print!("{}", "-".yellow());
}
return TestResult {
name: self.name.clone(),
strict,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
};
}
if verbose > 1 {
println!(
"`{}`{}: starting",
@ -139,183 +158,143 @@ impl Test {
}
let test_content = if strict {
format!("\"use strict\";\n{}", self.content)
Cow::Owned(format!("\"use strict\";\n{}", self.content))
} else {
self.content.to_string()
Cow::Borrowed(&*self.content)
};
let (result, result_text) = if !IGNORED.contains_any_flag(self.flags)
&& !IGNORED.contains_test(&self.name)
&& !IGNORED.contains_any_feature(&self.features)
&& (matches!(self.expected_outcome, Outcome::Positive)
|| matches!(
self.expected_outcome,
Outcome::Negative {
phase: Phase::Parse,
error_type: _,
}
)
|| matches!(
self.expected_outcome,
Outcome::Negative {
phase: Phase::Early,
error_type: _,
}
)
|| matches!(
self.expected_outcome,
Outcome::Negative {
phase: Phase::Runtime,
error_type: _,
}
)) {
let res = std::panic::catch_unwind(|| match self.expected_outcome {
Outcome::Positive => {
let mut context = Context::default();
let async_result = AsyncResult::default();
if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) {
return (false, e);
}
let result = std::panic::catch_unwind(|| match self.expected_outcome {
Outcome::Positive => {
let mut context = Context::default();
let async_result = AsyncResult::default();
// TODO: timeout
let value = match context.eval(&test_content) {
Ok(v) => v,
Err(e) => return (false, format!("Uncaught {e}")),
};
if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) {
return (false, e);
}
if let Err(e) = async_result.inner.borrow().as_ref() {
return (false, format!("Uncaught {e}"));
}
// TODO: timeout
let value = match context.eval(&*test_content) {
Ok(v) => v,
Err(e) => return (false, format!("Uncaught {e}")),
};
(true, value.display().to_string())
if let Err(e) = async_result.inner.borrow().as_ref() {
return (false, format!("Uncaught {e}"));
}
Outcome::Negative {
phase: Phase::Parse | Phase::Early,
(true, value.display().to_string())
}
Outcome::Negative {
phase: Phase::Parse | Phase::Early,
error_type,
} => {
assert_eq!(
error_type,
} => {
assert_eq!(
error_type,
ErrorType::SyntaxError,
"non-SyntaxError parsing/early error found in {}",
self.name
);
let mut context = Context::default();
match context.parse(&test_content) {
Ok(statement_list) => match context.compile(&statement_list) {
Ok(_) => (false, "StatementList compilation should fail".to_owned()),
Err(e) => (true, format!("Uncaught {e:?}")),
},
Err(e) => (true, format!("Uncaught {e}")),
}
ErrorType::SyntaxError,
"non-SyntaxError parsing/early error found in {}",
self.name
);
let mut context = Context::default();
match context.parse(&*test_content) {
Ok(statement_list) => match context.compile(&statement_list) {
Ok(_) => (false, "StatementList compilation should fail".to_owned()),
Err(e) => (true, format!("Uncaught {e:?}")),
},
Err(e) => (true, format!("Uncaught {e}")),
}
Outcome::Negative {
phase: Phase::Resolution,
error_type: _,
} => todo!("check module resolution errors"),
Outcome::Negative {
phase: Phase::Runtime,
error_type,
} => {
let mut context = Context::default();
if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) {
return (false, e);
}
let code = match Parser::new(test_content.as_bytes())
.parse_all(context.interner_mut())
.map_err(Into::into)
.and_then(|stmts| context.compile(&stmts))
{
Ok(code) => code,
Err(e) => return (false, format!("Uncaught {e}")),
};
// TODO: timeout
let e = match context.execute(code) {
Ok(res) => return (false, res.display().to_string()),
Err(e) => e,
};
if let Ok(e) = e.try_native(&mut context) {
match &e.kind {
JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
JsNativeErrorKind::Reference
if error_type == ErrorType::ReferenceError => {}
JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
_ => return (false, format!("Uncaught {e}")),
}
Outcome::Negative {
phase: Phase::Resolution,
error_type: _,
} => todo!("check module resolution errors"),
Outcome::Negative {
phase: Phase::Runtime,
error_type,
} => {
let mut context = Context::default();
if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) {
return (false, e);
}
let code = match Parser::new(test_content.as_bytes())
.parse_all(context.interner_mut())
.map_err(Into::into)
.and_then(|stmts| context.compile(&stmts))
{
Ok(code) => code,
Err(e) => return (false, format!("Uncaught {e}")),
};
// TODO: timeout
let e = match context.execute(code) {
Ok(res) => return (false, res.display().to_string()),
Err(e) => e,
};
if let Ok(e) = e.try_native(&mut context) {
match &e.kind {
JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
JsNativeErrorKind::Reference if error_type == ErrorType::ReferenceError => {
}
(true, format!("Uncaught {e}"))
} else {
let passed = e
.as_opaque()
.expect("try_native cannot fail if e is not opaque")
.as_object()
.and_then(|o| o.get("constructor", &mut context).ok())
.as_ref()
.and_then(JsValue::as_object)
.and_then(|o| o.get("name", &mut context).ok())
.as_ref()
.and_then(JsValue::as_string)
.map(|s| s == error_type.as_str())
.unwrap_or_default();
(passed, format!("Uncaught {e}"))
JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
_ => return (false, format!("Uncaught {e}")),
}
(true, format!("Uncaught {e}"))
} else {
let passed = e
.as_opaque()
.expect("try_native cannot fail if e is not opaque")
.as_object()
.and_then(|o| o.get("constructor", &mut context).ok())
.as_ref()
.and_then(JsValue::as_object)
.and_then(|o| o.get("name", &mut context).ok())
.as_ref()
.and_then(JsValue::as_string)
.map(|s| s == error_type.as_str())
.unwrap_or_default();
(passed, format!("Uncaught {e}"))
}
});
let result = res.map_or_else(
|_| {
eprintln!("last panic was on test \"{}\"", self.name);
(TestOutcomeResult::Panic, String::new())
},
|(res, text)| {
if res {
(TestOutcomeResult::Passed, text)
} else {
(TestOutcomeResult::Failed, text)
}
},
);
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
if matches!(result, (TestOutcomeResult::Passed, _)) {
"Passed".green()
} else if matches!(result, (TestOutcomeResult::Failed, _)) {
"Failed".red()
} else {
"⚠ Panic ⚠".red()
}
);
} else {
print!(
"{}",
if matches!(result, (TestOutcomeResult::Passed, _)) {
".".green()
} else {
"F".red()
}
);
}
});
let (result, result_text) = result.map_or_else(
|_| {
eprintln!("last panic was on test \"{}\"", self.name);
(TestOutcomeResult::Panic, String::new())
},
|(res, text)| {
if res {
(TestOutcomeResult::Passed, text)
} else {
(TestOutcomeResult::Failed, text)
}
},
);
result
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
if result == TestOutcomeResult::Passed {
"Passed".green()
} else if result == TestOutcomeResult::Failed {
"Failed".red()
} else {
"⚠ Panic ⚠".red()
}
);
} else {
if verbose > 1 {
println!(
"`{}`{}: {}",
self.name,
if strict { " (strict mode)" } else { "" },
"Ignored".yellow()
);
} else {
print!("{}", "-".yellow());
}
(TestOutcomeResult::Ignored, String::new())
};
print!(
"{}",
if result == TestOutcomeResult::Passed {
".".green()
} else {
"F".red()
}
);
}
if verbose > 2 {
println!(

223
boa_tester/src/main.rs
Unescape View File

@ -1,7 +1,7 @@
//! Test262 test runner
//!
//! This crate will run the full ECMAScript test suite (Test262) and report compliance of the
//! `boa` context.
//! `boa` wrap_err.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg",
html_favicon_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg"
@ -70,26 +70,34 @@ use self::{
read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
results::{compare_results, write_json},
};
use anyhow::{bail, Context};
use bitflags::bitflags;
use clap::{ArgAction, Parser, ValueHint};
use color_eyre::{
eyre::{bail, WrapErr},
Result,
};
use colored::Colorize;
use fxhash::{FxHashMap, FxHashSet};
use once_cell::sync::Lazy;
use read::ErrorType;
use serde::{Deserialize, Serialize};
use serde::{
de::{Unexpected, Visitor},
Deserialize, Deserializer, Serialize,
};
use std::{
fs,
fs::{self, File},
io::Read,
path::{Path, PathBuf},
};
/// Structure to allow defining ignored tests, features and files that should
/// be ignored even when reading.
#[derive(Debug)]
#[derive(Debug, Deserialize)]
struct Ignored {
#[serde(default)]
tests: FxHashSet<Box<str>>,
#[serde(default)]
features: FxHashSet<Box<str>>,
files: FxHashSet<Box<str>>,
#[serde(default = "TestFlags::empty")]
flags: TestFlags,
}
@ -102,16 +110,17 @@ impl Ignored {
/// Checks if the ignore list contains the given feature name in the list
/// of features to ignore.
pub(crate) fn contains_any_feature(&self, features: &[Box<str>]) -> bool {
features
.iter()
.any(|feature| self.features.contains(feature))
}
/// Checks if the ignore list contains the given file name in the list to
/// ignore from reading.
pub(crate) fn contains_file(&self, file: &str) -> bool {
self.files.contains(file)
pub(crate) fn contains_feature(&self, feature: &str) -> bool {
if self.features.contains(feature) {
return true;
}
// Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`.
// This ensures those are also ignored.
feature
.split('.')
.next()
.map(|feat| self.features.contains(feat))
.unwrap_or_default()
}
pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool {
@ -124,74 +133,11 @@ impl Default for Ignored {
Self {
tests: FxHashSet::default(),
features: FxHashSet::default(),
files: FxHashSet::default(),
flags: TestFlags::empty(),
}
}
}
/// List of ignored tests.
static IGNORED: Lazy<Ignored> = Lazy::new(|| {
let path = Path::new("test_ignore.txt");
if path.exists() {
let filtered = fs::read_to_string(path).expect("could not read test filters");
filtered
.lines()
.filter(|line| !line.is_empty() && !line.starts_with("//"))
.fold(Ignored::default(), |mut ign, line| {
// let mut line = line.to_owned();
if line.starts_with("file:") {
let file = line
.strip_prefix("file:")
.expect("prefix disappeared")
.trim()
.to_owned()
.into_boxed_str();
let test = if file.ends_with(".js") {
file.strip_suffix(".js")
.expect("suffix disappeared")
.to_owned()
.into_boxed_str()
} else {
file.clone()
};
ign.files.insert(file);
ign.tests.insert(test);
} else if line.starts_with("feature:") {
ign.features.insert(
line.strip_prefix("feature:")
.expect("prefix disappeared")
.trim()
.to_owned()
.into_boxed_str(),
);
} else if line.starts_with("flag:") {
let flag = line
.strip_prefix("flag:")
.expect("prefix disappeared")
.trim()
.parse::<TestFlag>()
.expect("invalid flag found");
ign.flags.insert(flag.into());
} else {
let mut test = line.trim();
if test
.rsplit('.')
.next()
.map(|ext| ext.eq_ignore_ascii_case("js"))
== Some(true)
{
test = test.strip_suffix(".js").expect("suffix disappeared");
}
ign.tests.insert(test.to_owned().into_boxed_str());
}
ign
})
} else {
Ignored::default()
}
});
/// Boa test262 tester
#[derive(Debug, Parser)]
#[command(author, version, about, name = "Boa test262 tester")]
@ -217,6 +163,10 @@ enum Cli {
/// Execute tests serially
#[arg(short, long)]
disable_parallelism: bool,
/// Path to a TOML file with the ignored tests, features, flags and/or files.
#[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)]
ignored: PathBuf,
},
/// Compare two test suite results.
Compare {
@ -235,7 +185,8 @@ enum Cli {
}
/// Program entry point.
fn main() {
fn main() -> Result<()> {
color_eyre::install()?;
match Cli::parse() {
Cli::Run {
verbose,
@ -243,23 +194,15 @@ fn main() {
suite,
output,
disable_parallelism,
} => {
if let Err(e) = run_test_suite(
verbose,
!disable_parallelism,
test262_path.as_path(),
suite.as_path(),
output.as_deref(),
) {
eprintln!("Error: {e}");
let mut src = e.source();
while let Some(e) = src {
eprintln!(" caused by: {e}");
src = e.source();
}
std::process::exit(1);
}
}
ignored: ignore,
} => run_test_suite(
verbose,
!disable_parallelism,
test262_path.as_path(),
suite.as_path(),
output.as_deref(),
ignore.as_path(),
),
Cli::Compare {
base,
new,
@ -275,24 +218,33 @@ fn run_test_suite(
test262_path: &Path,
suite: &Path,
output: Option<&Path>,
) -> anyhow::Result<()> {
ignored: &Path,
) -> Result<()> {
if let Some(path) = output {
if path.exists() {
if !path.is_dir() {
bail!("the output path must be a directory.");
}
} else {
fs::create_dir_all(path).context("could not create the output directory")?;
fs::create_dir_all(path).wrap_err("could not create the output directory")?;
}
}
let ignored = {
let mut input = String::new();
let mut f = File::open(ignored).wrap_err("could not open ignored tests file")?;
f.read_to_string(&mut input)
.wrap_err("could not read ignored tests file")?;
toml::from_str(&input).wrap_err("could not decode ignored tests file")?
};
if verbose != 0 {
println!("Loading the test suite...");
}
let harness = read_harness(test262_path).context("could not read harness")?;
let harness = read_harness(test262_path).wrap_err("could not read harness")?;
if suite.to_string_lossy().ends_with(".js") {
let test = read_test(&test262_path.join(suite)).with_context(|| {
let test = read_test(&test262_path.join(suite)).wrap_err_with(|| {
let suite = suite.display();
format!("could not read the test {suite}")
})?;
@ -304,7 +256,7 @@ fn run_test_suite(
println!();
} else {
let suite = read_suite(&test262_path.join(suite)).with_context(|| {
let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| {
let suite = suite.display();
format!("could not read the suite {suite}")
})?;
@ -332,7 +284,7 @@ fn run_test_suite(
);
write_json(results, output, verbose)
.context("could not write the results to the output JSON file")?;
.wrap_err("could not write the results to the output JSON file")?;
}
Ok(())
@ -419,6 +371,7 @@ struct Test {
includes: Box<[Box<str>]>,
locale: Locale,
content: Box<str>,
ignored: bool,
}
impl Test {
@ -440,15 +393,12 @@ impl Test {
includes: metadata.includes,
locale: metadata.locale,
content: content.into(),
ignored: false,
}
}
/// Sets the name of the test.
fn set_name<N>(&mut self, name: N)
where
N: Into<Box<str>>,
{
self.name = name.into();
fn set_ignored(&mut self) {
self.ignored = true;
}
}
@ -534,6 +484,59 @@ where
}
}
impl<'de> Deserialize<'de> for TestFlags {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct FlagsVisitor;
impl<'de> Visitor<'de> for FlagsVisitor {
type Value = TestFlags;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "a sequence of flags")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'de>,
{
let mut flags = TestFlags::empty();
while let Some(elem) = seq.next_element::<TestFlag>()? {
flags |= elem.into();
}
Ok(flags)
}
}
struct RawFlagsVisitor;
impl<'de> Visitor<'de> for RawFlagsVisitor {
type Value = TestFlags;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "a flags number")
}
fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
TestFlags::from_bits(v).ok_or_else(|| {
E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number")
})
}
}
if deserializer.is_human_readable() {
deserializer.deserialize_seq(FlagsVisitor)
} else {
deserializer.deserialize_u16(RawFlagsVisitor)
}
}
}
/// Phase for an error.
#[derive(Debug, Clone, Copy, Deserialize)]
#[serde(rename_all = "lowercase")]

92
boa_tester/src/read.rs
Unescape View File

@ -1,10 +1,15 @@
//! Module to read the list of test suites from disk.
use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED};
use anyhow::Context;
use crate::Ignored;
use super::{Harness, Locale, Phase, Test, TestSuite};
use color_eyre::{
eyre::{eyre, WrapErr},
Result,
};
use fxhash::FxHashMap;
use serde::Deserialize;
use std::{fs, io, path::Path, str::FromStr};
use std::{fs, io, path::Path};
/// Representation of the YAML metadata in Test262 tests.
#[derive(Debug, Clone, Deserialize)]
@ -77,31 +82,12 @@ pub(super) enum TestFlag {
NonDeterministic,
}
impl FromStr for TestFlag {
type Err = String; // TODO: improve error type.
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"onlyStrict" => Ok(Self::OnlyStrict),
"noStrict" => Ok(Self::NoStrict),
"module" => Ok(Self::Module),
"raw" => Ok(Self::Raw),
"async" => Ok(Self::Async),
"generated" => Ok(Self::Generated),
"CanBlockIsFalse" => Ok(Self::CanBlockIsFalse),
"CanBlockIsTrue" => Ok(Self::CanBlockIsTrue),
"non-deterministic" => Ok(Self::NonDeterministic),
_ => Err(format!("unknown test flag: {s}")),
}
}
}
/// Reads the Test262 defined bindings.
pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
pub(super) fn read_harness(test262_path: &Path) -> Result<Harness> {
let mut includes = FxHashMap::default();
for entry in
fs::read_dir(test262_path.join("harness")).context("error reading the harness directory")?
for entry in fs::read_dir(test262_path.join("harness"))
.wrap_err("error reading the harness directory")?
{
let entry = entry?;
let file_name = entry.file_name();
@ -112,7 +98,7 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
}
let content = fs::read_to_string(entry.path())
.with_context(|| format!("error reading the harnes/{file_name}"))?;
.wrap_err_with(|| format!("error reading the harnes/{file_name}"))?;
includes.insert(
file_name.into_owned().into_boxed_str(),
@ -120,13 +106,13 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
);
}
let assert = fs::read_to_string(test262_path.join("harness/assert.js"))
.context("error reading harnes/assert.js")?
.wrap_err("error reading harnes/assert.js")?
.into_boxed_str();
let sta = fs::read_to_string(test262_path.join("harness/sta.js"))
.context("error reading harnes/sta.js")?
.wrap_err("error reading harnes/sta.js")?
.into_boxed_str();
let doneprint_handle = fs::read_to_string(test262_path.join("harness/doneprintHandle.js"))
.context("error reading harnes/doneprintHandle.js")?
.wrap_err("error reading harnes/doneprintHandle.js")?
.into_boxed_str();
Ok(Harness {
@ -138,38 +124,54 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
}
/// Reads a test suite in the given path.
pub(super) fn read_suite(path: &Path) -> anyhow::Result<TestSuite> {
pub(super) fn read_suite(
path: &Path,
ignored: &Ignored,
mut ignore_suite: bool,
) -> Result<TestSuite> {
let name = path
.file_name()
.with_context(|| format!("test suite with no name found: {}", path.display()))?
.ok_or_else(|| eyre!(format!("test suite with no name found: {}", path.display())))?
.to_str()
.with_context(|| format!("non-UTF-8 suite name found: {}", path.display()))?;
.ok_or_else(|| eyre!(format!("non-UTF-8 suite name found: {}", path.display())))?;
ignore_suite |= ignored.contains_test(name);
let mut suites = Vec::new();
let mut tests = Vec::new();
// TODO: iterate in parallel
for entry in path.read_dir().context("retrieving entry")? {
for entry in path.read_dir().wrap_err("retrieving entry")? {
let entry = entry?;
if entry.file_type().context("retrieving file type")?.is_dir() {
suites.push(read_suite(entry.path().as_path()).with_context(|| {
let path = entry.path();
let suite = path.display();
format!("error reading sub-suite {suite}")
})?);
if entry.file_type().wrap_err("retrieving file type")?.is_dir() {
suites.push(
read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| {
let path = entry.path();
let suite = path.display();
format!("error reading sub-suite {suite}")
})?,
);
} else if entry.file_name().to_string_lossy().contains("_FIXTURE") {
continue;
} else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) {
let mut test = Test::default();
test.set_name(entry.file_name().to_string_lossy());
tests.push(test);
} else {
tests.push(read_test(entry.path().as_path()).with_context(|| {
let mut test = read_test(entry.path().as_path()).wrap_err_with(|| {
let path = entry.path();
let suite = path.display();
format!("error reading test {suite}")
})?);
})?;
if ignore_suite
|| ignored.contains_any_flag(test.flags)
|| ignored.contains_test(&test.name)
|| test
.features
.iter()
.any(|feat| ignored.contains_feature(feat))
{
test.set_ignored();
}
tests.push(test);
}
}

13
boa_tester/src/results.rs
Unescape View File

@ -1,4 +1,5 @@
use super::SuiteResult;
use color_eyre::{eyre::WrapErr, Result};
use serde::{Deserialize, Serialize};
use std::{
env, fs,
@ -208,16 +209,16 @@ fn update_gh_pages_repo(path: &Path, verbose: u8) {
}
/// Compares the results of two test suite runs.
pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result<()> {
let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
fs::File::open(base).expect("could not open the base results file"),
fs::File::open(base).wrap_err("could not open the base results file")?,
))
.expect("could not read the base results");
.wrap_err("could not read the base results")?;
let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
fs::File::open(new).expect("could not open the new results file"),
fs::File::open(new).wrap_err("could not open the new results file")?,
))
.expect("could not read the new results");
.wrap_err("could not read the new results")?;
let base_total = base_results.results.total as isize;
let new_total = new_results.results.total as isize;
@ -433,6 +434,8 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
}
}
}
Ok(())
}
/// Test differences.

45
test_ignore.toml
Unescape View File

@ -0,0 +1,45 @@
# Not implemented yet:
flags = ["module"]
features = [
# Non-implemented features:
"json-modules",
"SharedArrayBuffer",
"resizable-arraybuffer",
"Temporal",
"tail-call-optimization",
"ShadowRealm",
"FinalizationRegistry",
"Atomics",
"dynamic_import",
"decorators",
"array-grouping",
"IsHTMLDDA",
# Non-implemented Intl features
"intl-normative-optional",
"Intl.DurationFormat",
"Intl.NumberFormat-v3",
"Intl.NumberFormat-unified",
"Intl.ListFormat",
"Intl.DisplayNames",
"Intl.RelativeTimeFormat",
"Intl.Segmenter",
"Intl.Locale",
# Stage 3 proposals
# https://github.com/tc39/proposal-symbols-as-weakmap-keys
"symbols-as-weakmap-keys",
# Non-standard
"caller",
# RegExp tests that check individual codepoints.
# They are not useful considering the cpu time they waste.
"regexp-unicode-property-escapes",
]
# RegExp tests that check individual codepoints.
# They are not useful considering the cpu time they waste.
tests = ["CharacterClassEscapes"]

68
test_ignore.txt
Unescape View File

@ -1,68 +0,0 @@
// Not implemented yet:
flag:module
// Non-implemented features:
feature:json-modules
feature:SharedArrayBuffer
feature:resizable-arraybuffer
feature:Temporal
feature:tail-call-optimization
feature:ShadowRealm
feature:FinalizationRegistry
feature:FinalizationRegistry.prototype.cleanupSome
feature:Atomics
feature:dynamic_import
feature:decorators
feature:array-grouping
feature:IsHTMLDDA
// Non-implemented Intl features
feature:intl-normative-optional
feature:Intl.DurationFormat
feature:Intl.NumberFormat-v3
feature:Intl.NumberFormat-unified
feature:Intl.ListFormat
feature:Intl.DisplayNames
feature:Intl.RelativeTimeFormat
feature:Intl.Segmenter
feature:Intl.Locale
// Non-standard
feature:caller
// Stage 3 proposals
// https://github.com/tc39/proposal-symbols-as-weakmap-keys
feature:symbols-as-weakmap-keys
// These generate a stack overflow
tco-call
tco-member
// RegExp tests that check individual codepoints.
// They are not usefull in comparision to the cpu time they waste.
feature:regexp-unicode-property-escapes
character-class-non-whitespace-class-escape-plus-quantifier
character-class-non-whitespace-class-escape-plus-quantifier-flags-u
character-class-non-word-class-escape-flags-u
character-class-non-whitespace-class-escape
character-class-non-digit-class-escape-plus-quantifier-flags-u
character-class-non-word-class-escape
character-class-non-whitespace-class-escape-flags-u
character-class-digit-class-escape-flags-u
character-class-digit-class-escape
character-class-non-digit-class-escape-plus-quantifier
character-class-whitespace-class-escape-flags-u
character-class-whitespace-class-escape-plus-quantifier-flags-u
character-class-word-class-escape
character-class-digit-class-escape-plus-quantifier-flags-u
character-class-non-digit-class-escape
character-class-digit-class-escape-plus-quantifier
character-class-whitespace-class-escape
character-class-whitespace-class-escape-plus-quantifier
character-class-word-class-escape-plus-quantifier
character-class-non-word-class-escape-plus-quantifier
character-class-word-class-escape-flags-u
character-class-word-class-escape-plus-quantifier-flags-u
character-class-non-digit-class-escape-flags-u
character-class-non-word-class-escape-plus-quantifier-flags-u
Write Preview
Loading…
Cancel
Save