Cleanup `boa_tester` (#2440)

Just some quality changes to improve the maintainability of the tester: - Replaces `anyhow` with `color_eyre` to have a better output on errors/panics. - Changes the ignore file to a TOML file and replaces all parsing logic with the `toml` crate. - Adds a `ignored` field on all `Test`s to simplify run logic. - Replaces the global `IGNORED` with an `ignored` argument on the CLI.
2 years ago · 9c7b4d5531
8 changed files with 550 additions and 400 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -2,6 +2,21 @@
 # It is not intended for manual editing.
 version = 3
 [[package]]
 name = "addr2line"
 version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
 dependencies = [
 "gimli",
 ]
 [[package]]
 name = "adler"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 [[package]]
 name = "aho-corasick"
 version = "0.7.19"
@ -26,12 +41,6 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
 [[package]]
 name = "anyhow"
 version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
 [[package]]
 name = "arbitrary"
 version = "1.2.0"
@ -58,6 +67,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 [[package]]
 name = "backtrace"
 version = "0.3.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
 dependencies = [
 "addr2line",
 "cc",
 "cfg-if",
 "libc",
 "miniz_oxide",
 "object",
 "rustc-demangle",
 ]
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@ -210,13 +234,13 @@ dependencies = [
 name = "boa_tester"
 version = "0.16.0"
 dependencies = [
 "anyhow",
 "bitflags",
 "boa_engine",
 "boa_gc",
 "boa_interner",
 "boa_parser",
 "clap 4.0.25",
 "color-eyre",
 "colored",
 "fxhash",
 "once_cell",
@ -225,6 +249,7 @@ dependencies = [
 "serde",
 "serde_json",
 "serde_yaml",
 "toml",
 ]
 [[package]]
@ -394,6 +419,33 @@ dependencies = [
 "unicode-width",
 ]
 [[package]]
 name = "color-eyre"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204"
 dependencies = [
 "backtrace",
 "color-spantrace",
 "eyre",
 "indenter",
 "once_cell",
 "owo-colors",
 "tracing-error",
 ]
 [[package]]
 name = "color-spantrace"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
 dependencies = [
 "once_cell",
 "owo-colors",
 "tracing-core",
 "tracing-error",
 ]
 [[package]]
 name = "colored"
 version = "2.0.0"
@ -626,6 +678,16 @@ dependencies = [
 "str-buf",
 ]
 [[package]]
 name = "eyre"
 version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb"
 dependencies = [
 "indenter",
 "once_cell",
 ]
 [[package]]
 name = "fast-float"
 version = "0.2.0"
@ -692,6 +754,12 @@ dependencies = [
 "wasm-bindgen",
 ]
 [[package]]
 name = "gimli"
 version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
 [[package]]
 name = "half"
 version = "1.8.2"
@ -872,6 +940,12 @@ dependencies = [
 "icu_provider_blob",
 ]
 [[package]]
 name = "indenter"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
 [[package]]
 name = "indexmap"
 version = "1.9.1"
@ -1036,6 +1110,15 @@ dependencies = [
 "autocfg",
 ]
 [[package]]
 name = "miniz_oxide"
 version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
 dependencies = [
 "adler",
 ]
 [[package]]
 name = "nibble_vec"
 version = "0.1.0"
@ -1098,6 +1181,15 @@ dependencies = [
 "libc",
 ]
 [[package]]
 name = "object"
 version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
 dependencies = [
 "memchr",
 ]
 [[package]]
 name = "once_cell"
 version = "1.16.0"
@ -1116,6 +1208,12 @@ version = "6.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9"
 [[package]]
 name = "owo-colors"
 version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@ -1192,6 +1290,12 @@ dependencies = [
 "siphasher",
 ]
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 [[package]]
 name = "plotters"
 version = "0.3.4"
@ -1394,6 +1498,12 @@ dependencies = [
 "memchr",
 ]
 [[package]]
 name = "rustc-demangle"
 version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@ -1525,6 +1635,15 @@ dependencies = [
 "unsafe-libyaml",
 ]
 [[package]]
 name = "sharded-slab"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
 dependencies = [
 "lazy_static",
 ]
 [[package]]
 name = "siphasher"
 version = "0.3.10"
@ -1647,6 +1766,15 @@ dependencies = [
 "syn",
 ]
 [[package]]
 name = "thread_local"
 version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
 dependencies = [
 "once_cell",
 ]
 [[package]]
 name = "time"
 version = "0.1.44"
@ -1695,6 +1823,57 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 [[package]]
 name = "toml"
 version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
 dependencies = [
 "serde",
 ]
 [[package]]
 name = "tracing"
 version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
 dependencies = [
 "cfg-if",
 "pin-project-lite",
 "tracing-core",
 ]
 [[package]]
 name = "tracing-core"
 version = "0.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
 dependencies = [
 "once_cell",
 "valuable",
 ]
 [[package]]
 name = "tracing-error"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
 dependencies = [
 "tracing",
 "tracing-subscriber",
 ]
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
 dependencies = [
 "sharded-slab",
 "thread_local",
 "tracing-core",
 ]
 [[package]]
 name = "unicode-general-category"
 version = "0.6.0"
@ -1746,6 +1925,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
 [[package]]
 name = "valuable"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 [[package]]
 name = "version_check"
 version = "0.9.4"
--- a/boa_tester/Cargo.toml
+++ b/boa_tester/Cargo.toml
@ -26,4 +26,5 @@ once_cell = "1.16.0"
 colored = "2.0.0"
 fxhash = "0.2.1"
 rayon = "1.5.3"
-anyhow = "1.0.66"
+toml = "0.5.9"
 color-eyre = "0.6.2"
--- a/boa_tester/src/exec/mod.rs
+++ b/boa_tester/src/exec/mod.rs
@ -2,11 +2,12 @@
 mod js262;
 use std::borrow::Cow;
 use crate::read::ErrorType;
 use super::{
-    Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult,
+    Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, TestSuite,
    TestSuite, IGNORED,
 };
 use boa_engine::{
    builtins::JsArgs, object::FunctionBuilder, property::Attribute, Context, JsNativeErrorKind,
@ -130,6 +131,24 @@ impl Test {
    /// Runs the test once, in strict or non-strict mode
    fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
        if self.ignored {
            if verbose > 1 {
                println!(
                    "`{}`{}: {}",
                    self.name,
                    if strict { " (strict mode)" } else { "" },
                    "Ignored".yellow()
                );
            } else {
                print!("{}", "-".yellow());
            }
            return TestResult {
                name: self.name.clone(),
                strict,
                result: TestOutcomeResult::Ignored,
                result_text: Box::default(),
            };
        }
        if verbose > 1 {
            println!(
                "`{}`{}: starting",
@ -139,183 +158,143 @@ impl Test {
        }
        let test_content = if strict {
-            format!("\"use strict\";\n{}", self.content)
+            Cow::Owned(format!("\"use strict\";\n{}", self.content))
        } else {
-            self.content.to_string()
+            Cow::Borrowed(&*self.content)
        };
-        let (result, result_text) = if !IGNORED.contains_any_flag(self.flags)
+        let result = std::panic::catch_unwind(|| match self.expected_outcome {
-            && !IGNORED.contains_test(&self.name)
+            Outcome::Positive => {
-            && !IGNORED.contains_any_feature(&self.features)
+                let mut context = Context::default();
-            && (matches!(self.expected_outcome, Outcome::Positive)
+                let async_result = AsyncResult::default();
                || matches!(
                    self.expected_outcome,
                    Outcome::Negative {
                        phase: Phase::Parse,
                        error_type: _,
                    }
                )
                || matches!(
                    self.expected_outcome,
                    Outcome::Negative {
                        phase: Phase::Early,
                        error_type: _,
                    }
                )
                || matches!(
                    self.expected_outcome,
                    Outcome::Negative {
                        phase: Phase::Runtime,
                        error_type: _,
                    }
                )) {
            let res = std::panic::catch_unwind(|| match self.expected_outcome {
                Outcome::Positive => {
                    let mut context = Context::default();
                    let async_result = AsyncResult::default();
                    if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) {
                        return (false, e);
                    }
-                    // TODO: timeout
+                if let Err(e) = self.set_up_env(harness, &mut context, async_result.clone()) {
-                    let value = match context.eval(&test_content) {
+                    return (false, e);
-                        Ok(v) => v,
+                }
                        Err(e) => return (false, format!("Uncaught {e}")),
                    };
-                    if let Err(e) = async_result.inner.borrow().as_ref() {
+                // TODO: timeout
-                        return (false, format!("Uncaught {e}"));
+                let value = match context.eval(&*test_content) {
-                    }
+                    Ok(v) => v,
                    Err(e) => return (false, format!("Uncaught {e}")),
                };
-                    (true, value.display().to_string())
+                if let Err(e) = async_result.inner.borrow().as_ref() {
                    return (false, format!("Uncaught {e}"));
                }
-                Outcome::Negative {
+
-                    phase: Phase::Parse | Phase::Early,
+                (true, value.display().to_string())
            }
            Outcome::Negative {
                phase: Phase::Parse | Phase::Early,
                error_type,
            } => {
                assert_eq!(
                    error_type,
-                } => {
+                    ErrorType::SyntaxError,
-                    assert_eq!(
+                    "non-SyntaxError parsing/early error found in {}",
-                        error_type,
+                    self.name
-                        ErrorType::SyntaxError,
+                );
-                        "non-SyntaxError parsing/early error found in {}",
+
-                        self.name
+                let mut context = Context::default();
-                    );
+                match context.parse(&*test_content) {
-
+                    Ok(statement_list) => match context.compile(&statement_list) {
-                    let mut context = Context::default();
+                        Ok(_) => (false, "StatementList compilation should fail".to_owned()),
-                    match context.parse(&test_content) {
+                        Err(e) => (true, format!("Uncaught {e:?}")),
-                        Ok(statement_list) => match context.compile(&statement_list) {
+                    },
-                            Ok(_) => (false, "StatementList compilation should fail".to_owned()),
+                    Err(e) => (true, format!("Uncaught {e}")),
                            Err(e) => (true, format!("Uncaught {e:?}")),
                        },
                        Err(e) => (true, format!("Uncaught {e}")),
                    }
                }
-                Outcome::Negative {
+            }
-                    phase: Phase::Resolution,
+            Outcome::Negative {
-                    error_type: _,
+                phase: Phase::Resolution,
-                } => todo!("check module resolution errors"),
+                error_type: _,
-                Outcome::Negative {
+            } => todo!("check module resolution errors"),
-                    phase: Phase::Runtime,
+            Outcome::Negative {
-                    error_type,
+                phase: Phase::Runtime,
-                } => {
+                error_type,
-                    let mut context = Context::default();
+            } => {
-                    if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) {
+                let mut context = Context::default();
-                        return (false, e);
+                if let Err(e) = self.set_up_env(harness, &mut context, AsyncResult::default()) {
-                    }
+                    return (false, e);
-                    let code = match Parser::new(test_content.as_bytes())
+                }
-                        .parse_all(context.interner_mut())
+                let code = match Parser::new(test_content.as_bytes())
-                        .map_err(Into::into)
+                    .parse_all(context.interner_mut())
-                        .and_then(|stmts| context.compile(&stmts))
+                    .map_err(Into::into)
-                    {
+                    .and_then(|stmts| context.compile(&stmts))
-                        Ok(code) => code,
+                {
-                        Err(e) => return (false, format!("Uncaught {e}")),
+                    Ok(code) => code,
-                    };
+                    Err(e) => return (false, format!("Uncaught {e}")),
-
+                };
-                    // TODO: timeout
+
-                    let e = match context.execute(code) {
+                // TODO: timeout
-                        Ok(res) => return (false, res.display().to_string()),
+                let e = match context.execute(code) {
-                        Err(e) => e,
+                    Ok(res) => return (false, res.display().to_string()),
-                    };
+                    Err(e) => e,
-                    if let Ok(e) = e.try_native(&mut context) {
+                };
-                        match &e.kind {
+                if let Ok(e) = e.try_native(&mut context) {
-                            JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
+                    match &e.kind {
-                            JsNativeErrorKind::Reference
+                        JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
-                                if error_type == ErrorType::ReferenceError => {}
+                        JsNativeErrorKind::Reference if error_type == ErrorType::ReferenceError => {
                            JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
                            JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
                            _ => return (false, format!("Uncaught {e}")),
                        }
-                        (true, format!("Uncaught {e}"))
+                        JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
-                    } else {
+                        JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
-                        let passed = e
+                        _ => return (false, format!("Uncaught {e}")),
                            .as_opaque()
                            .expect("try_native cannot fail if e is not opaque")
                            .as_object()
                            .and_then(|o| o.get("constructor", &mut context).ok())
                            .as_ref()
                            .and_then(JsValue::as_object)
                            .and_then(|o| o.get("name", &mut context).ok())
                            .as_ref()
                            .and_then(JsValue::as_string)
                            .map(|s| s == error_type.as_str())
                            .unwrap_or_default();
                        (passed, format!("Uncaught {e}"))
                    }
                    (true, format!("Uncaught {e}"))
                } else {
                    let passed = e
                        .as_opaque()
                        .expect("try_native cannot fail if e is not opaque")
                        .as_object()
                        .and_then(|o| o.get("constructor", &mut context).ok())
                        .as_ref()
                        .and_then(JsValue::as_object)
                        .and_then(|o| o.get("name", &mut context).ok())
                        .as_ref()
                        .and_then(JsValue::as_string)
                        .map(|s| s == error_type.as_str())
                        .unwrap_or_default();
                    (passed, format!("Uncaught {e}"))
                }
            });
            let result = res.map_or_else(
                |_| {
                    eprintln!("last panic was on test \"{}\"", self.name);
                    (TestOutcomeResult::Panic, String::new())
                },
                |(res, text)| {
                    if res {
                        (TestOutcomeResult::Passed, text)
                    } else {
                        (TestOutcomeResult::Failed, text)
                    }
                },
            );
            if verbose > 1 {
                println!(
                    "`{}`{}: {}",
                    self.name,
                    if strict { " (strict mode)" } else { "" },
                    if matches!(result, (TestOutcomeResult::Passed, _)) {
                        "Passed".green()
                    } else if matches!(result, (TestOutcomeResult::Failed, _)) {
                        "Failed".red()
                    } else {
                        "⚠ Panic ⚠".red()
                    }
                );
            } else {
                print!(
                    "{}",
                    if matches!(result, (TestOutcomeResult::Passed, _)) {
                        ".".green()
                    } else {
                        "F".red()
                    }
                );
            }
        });
        let (result, result_text) = result.map_or_else(
            |_| {
                eprintln!("last panic was on test \"{}\"", self.name);
                (TestOutcomeResult::Panic, String::new())
            },
            |(res, text)| {
                if res {
                    (TestOutcomeResult::Passed, text)
                } else {
                    (TestOutcomeResult::Failed, text)
                }
            },
        );
-            result
+        if verbose > 1 {
            println!(
                "`{}`{}: {}",
                self.name,
                if strict { " (strict mode)" } else { "" },
                if result == TestOutcomeResult::Passed {
                    "Passed".green()
                } else if result == TestOutcomeResult::Failed {
                    "Failed".red()
                } else {
                    "⚠ Panic ⚠".red()
                }
            );
        } else {
-            if verbose > 1 {
+            print!(
-                println!(
+                "{}",
-                    "`{}`{}: {}",
+                if result == TestOutcomeResult::Passed {
-                    self.name,
+                    ".".green()
-                    if strict { " (strict mode)" } else { "" },
+                } else {
-                    "Ignored".yellow()
+                    "F".red()
-                );
+                }
-            } else {
+            );
-                print!("{}", "-".yellow());
+        }
            }
            (TestOutcomeResult::Ignored, String::new())
        };
        if verbose > 2 {
            println!(
--- a/boa_tester/src/main.rs
+++ b/boa_tester/src/main.rs
@ -1,7 +1,7 @@
 //! Test262 test runner
 //!
 //! This crate will run the full ECMAScript test suite (Test262) and report compliance of the
-//! `boa` context.
+//! `boa` wrap_err.
 #![doc(
    html_logo_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg",
    html_favicon_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg"
@ -70,26 +70,34 @@ use self::{
    read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
    results::{compare_results, write_json},
 };
 use anyhow::{bail, Context};
 use bitflags::bitflags;
 use clap::{ArgAction, Parser, ValueHint};
 use color_eyre::{
    eyre::{bail, WrapErr},
    Result,
 };
 use colored::Colorize;
 use fxhash::{FxHashMap, FxHashSet};
 use once_cell::sync::Lazy;
 use read::ErrorType;
-use serde::{Deserialize, Serialize};
+use serde::{
    de::{Unexpected, Visitor},
    Deserialize, Deserializer, Serialize,
 };
 use std::{
-    fs,
+    fs::{self, File},
    io::Read,
    path::{Path, PathBuf},
 };
 /// Structure to allow defining ignored tests, features and files that should
 /// be ignored even when reading.
-#[derive(Debug)]
+#[derive(Debug, Deserialize)]
 struct Ignored {
    #[serde(default)]
    tests: FxHashSet<Box<str>>,
    #[serde(default)]
    features: FxHashSet<Box<str>>,
-    files: FxHashSet<Box<str>>,
+    #[serde(default = "TestFlags::empty")]
    flags: TestFlags,
 }
@ -102,16 +110,17 @@ impl Ignored {
    /// Checks if the ignore list contains the given feature name in the list
    /// of features to ignore.
-    pub(crate) fn contains_any_feature(&self, features: &[Box<str>]) -> bool {
+    pub(crate) fn contains_feature(&self, feature: &str) -> bool {
-        features
+        if self.features.contains(feature) {
-            .iter()
+            return true;
-            .any(|feature| self.features.contains(feature))
+        }
-    }
+        // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`.
-
+        // This ensures those are also ignored.
-    /// Checks if the ignore list contains the given file name in the list to
+        feature
-    /// ignore from reading.
+            .split('.')
-    pub(crate) fn contains_file(&self, file: &str) -> bool {
+            .next()
-        self.files.contains(file)
+            .map(|feat| self.features.contains(feat))
            .unwrap_or_default()
    }
    pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool {
@ -124,74 +133,11 @@ impl Default for Ignored {
        Self {
            tests: FxHashSet::default(),
            features: FxHashSet::default(),
            files: FxHashSet::default(),
            flags: TestFlags::empty(),
        }
    }
 }
 /// List of ignored tests.
 static IGNORED: Lazy<Ignored> = Lazy::new(|| {
    let path = Path::new("test_ignore.txt");
    if path.exists() {
        let filtered = fs::read_to_string(path).expect("could not read test filters");
        filtered
            .lines()
            .filter(|line| !line.is_empty() && !line.starts_with("//"))
            .fold(Ignored::default(), |mut ign, line| {
                // let mut line = line.to_owned();
                if line.starts_with("file:") {
                    let file = line
                        .strip_prefix("file:")
                        .expect("prefix disappeared")
                        .trim()
                        .to_owned()
                        .into_boxed_str();
                    let test = if file.ends_with(".js") {
                        file.strip_suffix(".js")
                            .expect("suffix disappeared")
                            .to_owned()
                            .into_boxed_str()
                    } else {
                        file.clone()
                    };
                    ign.files.insert(file);
                    ign.tests.insert(test);
                } else if line.starts_with("feature:") {
                    ign.features.insert(
                        line.strip_prefix("feature:")
                            .expect("prefix disappeared")
                            .trim()
                            .to_owned()
                            .into_boxed_str(),
                    );
                } else if line.starts_with("flag:") {
                    let flag = line
                        .strip_prefix("flag:")
                        .expect("prefix disappeared")
                        .trim()
                        .parse::<TestFlag>()
                        .expect("invalid flag found");
                    ign.flags.insert(flag.into());
                } else {
                    let mut test = line.trim();
                    if test
                        .rsplit('.')
                        .next()
                        .map(|ext| ext.eq_ignore_ascii_case("js"))
                        == Some(true)
                    {
                        test = test.strip_suffix(".js").expect("suffix disappeared");
                    }
                    ign.tests.insert(test.to_owned().into_boxed_str());
                }
                ign
            })
    } else {
        Ignored::default()
    }
 });
 /// Boa test262 tester
 #[derive(Debug, Parser)]
 #[command(author, version, about, name = "Boa test262 tester")]
@ -217,6 +163,10 @@ enum Cli {
        /// Execute tests serially
        #[arg(short, long)]
        disable_parallelism: bool,
        /// Path to a TOML file with the ignored tests, features, flags and/or files.
        #[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)]
        ignored: PathBuf,
    },
    /// Compare two test suite results.
    Compare {
@ -235,7 +185,8 @@ enum Cli {
 }
 /// Program entry point.
-fn main() {
+fn main() -> Result<()> {
    color_eyre::install()?;
    match Cli::parse() {
        Cli::Run {
            verbose,
@ -243,23 +194,15 @@ fn main() {
            suite,
            output,
            disable_parallelism,
-        } => {
+            ignored: ignore,
-            if let Err(e) = run_test_suite(
+        } => run_test_suite(
-                verbose,
+            verbose,
-                !disable_parallelism,
+            !disable_parallelism,
-                test262_path.as_path(),
+            test262_path.as_path(),
-                suite.as_path(),
+            suite.as_path(),
-                output.as_deref(),
+            output.as_deref(),
-            ) {
+            ignore.as_path(),
-                eprintln!("Error: {e}");
+        ),
                let mut src = e.source();
                while let Some(e) = src {
                    eprintln!("    caused by: {e}");
                    src = e.source();
                }
                std::process::exit(1);
            }
        }
        Cli::Compare {
            base,
            new,
@ -275,24 +218,33 @@ fn run_test_suite(
    test262_path: &Path,
    suite: &Path,
    output: Option<&Path>,
-) -> anyhow::Result<()> {
+    ignored: &Path,
 ) -> Result<()> {
    if let Some(path) = output {
        if path.exists() {
            if !path.is_dir() {
                bail!("the output path must be a directory.");
            }
        } else {
-            fs::create_dir_all(path).context("could not create the output directory")?;
+            fs::create_dir_all(path).wrap_err("could not create the output directory")?;
        }
    }
    let ignored = {
        let mut input = String::new();
        let mut f = File::open(ignored).wrap_err("could not open ignored tests file")?;
        f.read_to_string(&mut input)
            .wrap_err("could not read ignored tests file")?;
        toml::from_str(&input).wrap_err("could not decode ignored tests file")?
    };
    if verbose != 0 {
        println!("Loading the test suite...");
    }
-    let harness = read_harness(test262_path).context("could not read harness")?;
+    let harness = read_harness(test262_path).wrap_err("could not read harness")?;
    if suite.to_string_lossy().ends_with(".js") {
-        let test = read_test(&test262_path.join(suite)).with_context(|| {
+        let test = read_test(&test262_path.join(suite)).wrap_err_with(|| {
            let suite = suite.display();
            format!("could not read the test {suite}")
        })?;
@ -304,7 +256,7 @@ fn run_test_suite(
        println!();
    } else {
-        let suite = read_suite(&test262_path.join(suite)).with_context(|| {
+        let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| {
            let suite = suite.display();
            format!("could not read the suite {suite}")
        })?;
@ -332,7 +284,7 @@ fn run_test_suite(
        );
        write_json(results, output, verbose)
-            .context("could not write the results to the output JSON file")?;
+            .wrap_err("could not write the results to the output JSON file")?;
    }
    Ok(())
@ -419,6 +371,7 @@ struct Test {
    includes: Box<[Box<str>]>,
    locale: Locale,
    content: Box<str>,
    ignored: bool,
 }
 impl Test {
@ -440,15 +393,12 @@ impl Test {
            includes: metadata.includes,
            locale: metadata.locale,
            content: content.into(),
            ignored: false,
        }
    }
-    /// Sets the name of the test.
+    fn set_ignored(&mut self) {
-    fn set_name<N>(&mut self, name: N)
+        self.ignored = true;
    where
        N: Into<Box<str>>,
    {
        self.name = name.into();
    }
 }
@ -534,6 +484,59 @@ where
    }
 }
 impl<'de> Deserialize<'de> for TestFlags {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        struct FlagsVisitor;
        impl<'de> Visitor<'de> for FlagsVisitor {
            type Value = TestFlags;
            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(formatter, "a sequence of flags")
            }
            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
            where
                A: serde::de::SeqAccess<'de>,
            {
                let mut flags = TestFlags::empty();
                while let Some(elem) = seq.next_element::<TestFlag>()? {
                    flags |= elem.into();
                }
                Ok(flags)
            }
        }
        struct RawFlagsVisitor;
        impl<'de> Visitor<'de> for RawFlagsVisitor {
            type Value = TestFlags;
            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(formatter, "a flags number")
            }
            fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>
            where
                E: serde::de::Error,
            {
                TestFlags::from_bits(v).ok_or_else(|| {
                    E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number")
                })
            }
        }
        if deserializer.is_human_readable() {
            deserializer.deserialize_seq(FlagsVisitor)
        } else {
            deserializer.deserialize_u16(RawFlagsVisitor)
        }
    }
 }
 /// Phase for an error.
 #[derive(Debug, Clone, Copy, Deserialize)]
 #[serde(rename_all = "lowercase")]
--- a/boa_tester/src/read.rs
+++ b/boa_tester/src/read.rs
@ -1,10 +1,15 @@
 //! Module to read the list of test suites from disk.
-use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED};
+use crate::Ignored;
-use anyhow::Context;
+
 use super::{Harness, Locale, Phase, Test, TestSuite};
 use color_eyre::{
    eyre::{eyre, WrapErr},
    Result,
 };
 use fxhash::FxHashMap;
 use serde::Deserialize;
-use std::{fs, io, path::Path, str::FromStr};
+use std::{fs, io, path::Path};
 /// Representation of the YAML metadata in Test262 tests.
 #[derive(Debug, Clone, Deserialize)]
@ -77,31 +82,12 @@ pub(super) enum TestFlag {
    NonDeterministic,
 }
 impl FromStr for TestFlag {
    type Err = String; // TODO: improve error type.
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s {
            "onlyStrict" => Ok(Self::OnlyStrict),
            "noStrict" => Ok(Self::NoStrict),
            "module" => Ok(Self::Module),
            "raw" => Ok(Self::Raw),
            "async" => Ok(Self::Async),
            "generated" => Ok(Self::Generated),
            "CanBlockIsFalse" => Ok(Self::CanBlockIsFalse),
            "CanBlockIsTrue" => Ok(Self::CanBlockIsTrue),
            "non-deterministic" => Ok(Self::NonDeterministic),
            _ => Err(format!("unknown test flag: {s}")),
        }
    }
 }
 /// Reads the Test262 defined bindings.
-pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
+pub(super) fn read_harness(test262_path: &Path) -> Result<Harness> {
    let mut includes = FxHashMap::default();
-    for entry in
+    for entry in fs::read_dir(test262_path.join("harness"))
-        fs::read_dir(test262_path.join("harness")).context("error reading the harness directory")?
+        .wrap_err("error reading the harness directory")?
    {
        let entry = entry?;
        let file_name = entry.file_name();
@ -112,7 +98,7 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
        }
        let content = fs::read_to_string(entry.path())
-            .with_context(|| format!("error reading the harnes/{file_name}"))?;
+            .wrap_err_with(|| format!("error reading the harnes/{file_name}"))?;
        includes.insert(
            file_name.into_owned().into_boxed_str(),
@ -120,13 +106,13 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
        );
    }
    let assert = fs::read_to_string(test262_path.join("harness/assert.js"))
-        .context("error reading harnes/assert.js")?
+        .wrap_err("error reading harnes/assert.js")?
        .into_boxed_str();
    let sta = fs::read_to_string(test262_path.join("harness/sta.js"))
-        .context("error reading harnes/sta.js")?
+        .wrap_err("error reading harnes/sta.js")?
        .into_boxed_str();
    let doneprint_handle = fs::read_to_string(test262_path.join("harness/doneprintHandle.js"))
-        .context("error reading harnes/doneprintHandle.js")?
+        .wrap_err("error reading harnes/doneprintHandle.js")?
        .into_boxed_str();
    Ok(Harness {
@ -138,38 +124,54 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
 }
 /// Reads a test suite in the given path.
-pub(super) fn read_suite(path: &Path) -> anyhow::Result<TestSuite> {
+pub(super) fn read_suite(
    path: &Path,
    ignored: &Ignored,
    mut ignore_suite: bool,
 ) -> Result<TestSuite> {
    let name = path
        .file_name()
-        .with_context(|| format!("test suite with no name found: {}", path.display()))?
+        .ok_or_else(|| eyre!(format!("test suite with no name found: {}", path.display())))?
        .to_str()
-        .with_context(|| format!("non-UTF-8 suite name found: {}", path.display()))?;
+        .ok_or_else(|| eyre!(format!("non-UTF-8 suite name found: {}", path.display())))?;
    ignore_suite |= ignored.contains_test(name);
    let mut suites = Vec::new();
    let mut tests = Vec::new();
    // TODO: iterate in parallel
-    for entry in path.read_dir().context("retrieving entry")? {
+    for entry in path.read_dir().wrap_err("retrieving entry")? {
        let entry = entry?;
-        if entry.file_type().context("retrieving file type")?.is_dir() {
+        if entry.file_type().wrap_err("retrieving file type")?.is_dir() {
-            suites.push(read_suite(entry.path().as_path()).with_context(|| {
+            suites.push(
-                let path = entry.path();
+                read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| {
-                let suite = path.display();
+                    let path = entry.path();
-                format!("error reading sub-suite {suite}")
+                    let suite = path.display();
-            })?);
+                    format!("error reading sub-suite {suite}")
                })?,
            );
        } else if entry.file_name().to_string_lossy().contains("_FIXTURE") {
            continue;
        } else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) {
            let mut test = Test::default();
            test.set_name(entry.file_name().to_string_lossy());
            tests.push(test);
        } else {
-            tests.push(read_test(entry.path().as_path()).with_context(|| {
+            let mut test = read_test(entry.path().as_path()).wrap_err_with(|| {
                let path = entry.path();
                let suite = path.display();
                format!("error reading test {suite}")
-            })?);
+            })?;
            if ignore_suite
                || ignored.contains_any_flag(test.flags)
                || ignored.contains_test(&test.name)
                || test
                    .features
                    .iter()
                    .any(|feat| ignored.contains_feature(feat))
            {
                test.set_ignored();
            }
            tests.push(test);
        }
    }
--- a/boa_tester/src/results.rs
+++ b/boa_tester/src/results.rs
@ -1,4 +1,5 @@
 use super::SuiteResult;
 use color_eyre::{eyre::WrapErr, Result};
 use serde::{Deserialize, Serialize};
 use std::{
    env, fs,
@ -208,16 +209,16 @@ fn update_gh_pages_repo(path: &Path, verbose: u8) {
 }
 /// Compares the results of two test suite runs.
-pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
+pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result<()> {
    let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
-        fs::File::open(base).expect("could not open the base results file"),
+        fs::File::open(base).wrap_err("could not open the base results file")?,
    ))
-    .expect("could not read the base results");
+    .wrap_err("could not read the base results")?;
    let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
-        fs::File::open(new).expect("could not open the new results file"),
+        fs::File::open(new).wrap_err("could not open the new results file")?,
    ))
-    .expect("could not read the new results");
+    .wrap_err("could not read the new results")?;
    let base_total = base_results.results.total as isize;
    let new_total = new_results.results.total as isize;
@ -433,6 +434,8 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
            }
        }
    }
    Ok(())
 }
 /// Test differences.
--- a/test_ignore.toml
+++ b/test_ignore.toml
@ -0,0 +1,45 @@
 # Not implemented yet:
 flags = ["module"]
 features = [
    # Non-implemented features:
    "json-modules",
    "SharedArrayBuffer",
    "resizable-arraybuffer",
    "Temporal",
    "tail-call-optimization",
    "ShadowRealm",
    "FinalizationRegistry",
    "Atomics",
    "dynamic_import",
    "decorators",
    "array-grouping",
    "IsHTMLDDA",
    # Non-implemented Intl features
    "intl-normative-optional",
    "Intl.DurationFormat",
    "Intl.NumberFormat-v3",
    "Intl.NumberFormat-unified",
    "Intl.ListFormat",
    "Intl.DisplayNames",
    "Intl.RelativeTimeFormat",
    "Intl.Segmenter",
    "Intl.Locale",
    # Stage 3 proposals
    # https://github.com/tc39/proposal-symbols-as-weakmap-keys
    "symbols-as-weakmap-keys",
    # Non-standard
    "caller",
    # RegExp tests that check individual codepoints.
    # They are not useful considering the cpu time they waste.
    "regexp-unicode-property-escapes",
 ]
 # RegExp tests that check individual codepoints.
 # They are not useful considering the cpu time they waste.
 tests = ["CharacterClassEscapes"]
--- a/test_ignore.txt
+++ b/test_ignore.txt
@ -1,68 +0,0 @@
 // Not implemented yet:
 flag:module
 // Non-implemented features:
 feature:json-modules
 feature:SharedArrayBuffer
 feature:resizable-arraybuffer
 feature:Temporal
 feature:tail-call-optimization
 feature:ShadowRealm
 feature:FinalizationRegistry
 feature:FinalizationRegistry.prototype.cleanupSome
 feature:Atomics
 feature:dynamic_import
 feature:decorators
 feature:array-grouping
 feature:IsHTMLDDA
 // Non-implemented Intl features
 feature:intl-normative-optional
 feature:Intl.DurationFormat
 feature:Intl.NumberFormat-v3
 feature:Intl.NumberFormat-unified
 feature:Intl.ListFormat
 feature:Intl.DisplayNames
 feature:Intl.RelativeTimeFormat
 feature:Intl.Segmenter
 feature:Intl.Locale
 // Non-standard
 feature:caller
 // Stage 3 proposals
 // https://github.com/tc39/proposal-symbols-as-weakmap-keys
 feature:symbols-as-weakmap-keys
 // These generate a stack overflow
 tco-call
 tco-member
 // RegExp tests that check individual codepoints.
 // They are not usefull in comparision to the cpu time they waste.
 feature:regexp-unicode-property-escapes
 character-class-non-whitespace-class-escape-plus-quantifier
 character-class-non-whitespace-class-escape-plus-quantifier-flags-u
 character-class-non-word-class-escape-flags-u
 character-class-non-whitespace-class-escape
 character-class-non-digit-class-escape-plus-quantifier-flags-u
 character-class-non-word-class-escape
 character-class-non-whitespace-class-escape-flags-u
 character-class-digit-class-escape-flags-u
 character-class-digit-class-escape
 character-class-non-digit-class-escape-plus-quantifier
 character-class-whitespace-class-escape-flags-u
 character-class-whitespace-class-escape-plus-quantifier-flags-u
 character-class-word-class-escape
 character-class-digit-class-escape-plus-quantifier-flags-u
 character-class-non-digit-class-escape
 character-class-digit-class-escape-plus-quantifier
 character-class-whitespace-class-escape
 character-class-whitespace-class-escape-plus-quantifier
 character-class-word-class-escape-plus-quantifier
 character-class-non-word-class-escape-plus-quantifier
 character-class-word-class-escape-flags-u
 character-class-word-class-escape-plus-quantifier-flags-u
 character-class-non-digit-class-escape-flags-u
 character-class-non-word-class-escape-plus-quantifier-flags-u