Cleanup `boa_tester` (#2440)

Just some quality changes to improve the maintainability of the tester: - Replaces `anyhow` with `color_eyre` to have a better output on errors/panics. - Changes the ignore file to a TOML file and replaces all parsing logic with the `toml` crate. - Adds a `ignored` field on all `Test`s to simplify run logic. - Replaces the global `IGNORED` with an `ignored` argument on the CLI.
2 years ago · 9c7b4d5531
8 changed files with 550 additions and 400 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -2,6 +2,21 @@
 # It is not intended for manual editing.
 version = 3

+[[package]]
+name = "addr2line"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.19"
@ -26,12 +41,6 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"

-[[package]]
-name = "anyhow"
-version = "1.0.66"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
-
 [[package]]
 name = "arbitrary"
 version = "1.2.0"
@ -58,6 +67,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

+[[package]]
+name = "backtrace"
+version = "0.3.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@ -210,13 +234,13 @@ dependencies = [
 name = "boa_tester"
 version = "0.16.0"
 dependencies = [
- "anyhow",
 "bitflags",
 "boa_engine",
 "boa_gc",
 "boa_interner",
 "boa_parser",
 "clap 4.0.25",
+ "color-eyre",
 "colored",
 "fxhash",
 "once_cell",
@ -225,6 +249,7 @@ dependencies = [
 "serde",
 "serde_json",
 "serde_yaml",
+ "toml",
 ]

 [[package]]
@ -394,6 +419,33 @@ dependencies = [
 "unicode-width",
 ]

+[[package]]
+name = "color-eyre"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204"
+dependencies = [
+ "backtrace",
+ "color-spantrace",
+ "eyre",
+ "indenter",
+ "once_cell",
+ "owo-colors",
+ "tracing-error",
+]
+
+[[package]]
+name = "color-spantrace"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
+dependencies = [
+ "once_cell",
+ "owo-colors",
+ "tracing-core",
+ "tracing-error",
+]
+
 [[package]]
 name = "colored"
 version = "2.0.0"
@ -626,6 +678,16 @@ dependencies = [
 "str-buf",
 ]

+[[package]]
+name = "eyre"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb"
+dependencies = [
+ "indenter",
+ "once_cell",
+]
+
 [[package]]
 name = "fast-float"
 version = "0.2.0"
@ -692,6 +754,12 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "gimli"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
+
 [[package]]
 name = "half"
 version = "1.8.2"
@ -872,6 +940,12 @@ dependencies = [
 "icu_provider_blob",
 ]

+[[package]]
+name = "indenter"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
+
 [[package]]
 name = "indexmap"
 version = "1.9.1"
@ -1036,6 +1110,15 @@ dependencies = [
 "autocfg",
 ]

+[[package]]
+name = "miniz_oxide"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "nibble_vec"
 version = "0.1.0"
@ -1098,6 +1181,15 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "object"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.16.0"
@ -1116,6 +1208,12 @@ version = "6.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9"

+[[package]]
+name = "owo-colors"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@ -1192,6 +1290,12 @@ dependencies = [
 "siphasher",
 ]

+[[package]]
+name = "pin-project-lite"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+
 [[package]]
 name = "plotters"
 version = "0.3.4"
@ -1394,6 +1498,12 @@ dependencies = [
 "memchr",
 ]

+[[package]]
+name = "rustc-demangle"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@ -1525,6 +1635,15 @@ dependencies = [
 "unsafe-libyaml",
 ]

+[[package]]
+name = "sharded-slab"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
+dependencies = [
+ "lazy_static",
+]
+
 [[package]]
 name = "siphasher"
 version = "0.3.10"
@ -1647,6 +1766,15 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "thread_local"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+dependencies = [
+ "once_cell",
+]
+
 [[package]]
 name = "time"
 version = "0.1.44"
@ -1695,6 +1823,57 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"

+[[package]]
+name = "toml"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "tracing"
+version = "0.1.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+dependencies = [
+ "cfg-if",
+ "pin-project-lite",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
+dependencies = [
+ "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-error"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
+dependencies = [
+ "tracing",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
+dependencies = [
+ "sharded-slab",
+ "thread_local",
+ "tracing-core",
+]
+
 [[package]]
 name = "unicode-general-category"
 version = "0.6.0"
@ -1746,6 +1925,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"

+[[package]]
+name = "valuable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
--- a/boa_tester/Cargo.toml
+++ b/boa_tester/Cargo.toml
@ -26,4 +26,5 @@ once_cell = "1.16.0"
 colored = "2.0.0"
 fxhash = "0.2.1"
 rayon = "1.5.3"
-anyhow = "1.0.66"
+toml = "0.5.9"
+color-eyre = "0.6.2"
--- a/boa_tester/src/exec/mod.rs
+++ b/boa_tester/src/exec/mod.rs
@ -2,11 +2,12 @@

 mod js262;

+use std::borrow::Cow;
+
 use crate::read::ErrorType;

 use super::{
-    Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult,
-    TestSuite, IGNORED,
+    Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, TestSuite,
 };
 use boa_engine::{
    builtins::JsArgs, object::FunctionBuilder, property::Attribute, Context, JsNativeErrorKind,
@ -130,6 +131,24 @@ impl Test {

    /// Runs the test once, in strict or non-strict mode
    fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
+        if self.ignored {
+            if verbose > 1 {
+                println!(
+                    "`{}`{}: {}",
+                    self.name,
+                    if strict { " (strict mode)" } else { "" },
+                    "Ignored".yellow()
+                );
+            } else {
+                print!("{}", "-".yellow());
+            }
+            return TestResult {
+                name: self.name.clone(),
+                strict,
+                result: TestOutcomeResult::Ignored,
+                result_text: Box::default(),
+            };
+        }
        if verbose > 1 {
            println!(
                "`{}`{}: starting",
@ -139,37 +158,12 @@ impl Test {
        }

        let test_content = if strict {
-            format!("\"use strict\";\n{}", self.content)
+            Cow::Owned(format!("\"use strict\";\n{}", self.content))
        } else {
-            self.content.to_string()
+            Cow::Borrowed(&*self.content)
        };

-        let (result, result_text) = if !IGNORED.contains_any_flag(self.flags)
-            && !IGNORED.contains_test(&self.name)
-            && !IGNORED.contains_any_feature(&self.features)
-            && (matches!(self.expected_outcome, Outcome::Positive)
-                || matches!(
-                    self.expected_outcome,
-                    Outcome::Negative {
-                        phase: Phase::Parse,
-                        error_type: _,
-                    }
-                )
-                || matches!(
-                    self.expected_outcome,
-                    Outcome::Negative {
-                        phase: Phase::Early,
-                        error_type: _,
-                    }
-                )
-                || matches!(
-                    self.expected_outcome,
-                    Outcome::Negative {
-                        phase: Phase::Runtime,
-                        error_type: _,
-                    }
-                )) {
-            let res = std::panic::catch_unwind(|| match self.expected_outcome {
+        let result = std::panic::catch_unwind(|| match self.expected_outcome {
            Outcome::Positive => {
                let mut context = Context::default();
                let async_result = AsyncResult::default();
@ -179,7 +173,7 @@ impl Test {
                }

                // TODO: timeout
-                    let value = match context.eval(&test_content) {
+                let value = match context.eval(&*test_content) {
                    Ok(v) => v,
                    Err(e) => return (false, format!("Uncaught {e}")),
                };
@ -202,7 +196,7 @@ impl Test {
                );

                let mut context = Context::default();
-                    match context.parse(&test_content) {
+                match context.parse(&*test_content) {
                    Ok(statement_list) => match context.compile(&statement_list) {
                        Ok(_) => (false, "StatementList compilation should fail".to_owned()),
                        Err(e) => (true, format!("Uncaught {e:?}")),
@ -239,8 +233,8 @@ impl Test {
                if let Ok(e) = e.try_native(&mut context) {
                    match &e.kind {
                        JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
-                            JsNativeErrorKind::Reference
-                                if error_type == ErrorType::ReferenceError => {}
+                        JsNativeErrorKind::Reference if error_type == ErrorType::ReferenceError => {
+                        }
                        JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
                        JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
                        _ => return (false, format!("Uncaught {e}")),
@ -264,7 +258,7 @@ impl Test {
            }
        });

-            let result = res.map_or_else(
+        let (result, result_text) = result.map_or_else(
            |_| {
                eprintln!("last panic was on test \"{}\"", self.name);
                (TestOutcomeResult::Panic, String::new())
@ -283,9 +277,9 @@ impl Test {
                "`{}`{}: {}",
                self.name,
                if strict { " (strict mode)" } else { "" },
-                    if matches!(result, (TestOutcomeResult::Passed, _)) {
+                if result == TestOutcomeResult::Passed {
                    "Passed".green()
-                    } else if matches!(result, (TestOutcomeResult::Failed, _)) {
+                } else if result == TestOutcomeResult::Failed {
                    "Failed".red()
                } else {
                    "⚠ Panic ⚠".red()
@ -294,7 +288,7 @@ impl Test {
        } else {
            print!(
                "{}",
-                    if matches!(result, (TestOutcomeResult::Passed, _)) {
+                if result == TestOutcomeResult::Passed {
                    ".".green()
                } else {
                    "F".red()
@ -302,21 +296,6 @@ impl Test {
            );
        }

-            result
-        } else {
-            if verbose > 1 {
-                println!(
-                    "`{}`{}: {}",
-                    self.name,
-                    if strict { " (strict mode)" } else { "" },
-                    "Ignored".yellow()
-                );
-            } else {
-                print!("{}", "-".yellow());
-            }
-            (TestOutcomeResult::Ignored, String::new())
-        };
-
        if verbose > 2 {
            println!(
                "`{}`{}: result text",
--- a/boa_tester/src/main.rs
+++ b/boa_tester/src/main.rs
@ -1,7 +1,7 @@
 //! Test262 test runner
 //!
 //! This crate will run the full ECMAScript test suite (Test262) and report compliance of the
-//! `boa` context.
+//! `boa` wrap_err.
 #![doc(
    html_logo_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg",
    html_favicon_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg"
@ -70,26 +70,34 @@ use self::{
    read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
    results::{compare_results, write_json},
 };
-use anyhow::{bail, Context};
 use bitflags::bitflags;
 use clap::{ArgAction, Parser, ValueHint};
+use color_eyre::{
+    eyre::{bail, WrapErr},
+    Result,
+};
 use colored::Colorize;
 use fxhash::{FxHashMap, FxHashSet};
-use once_cell::sync::Lazy;
 use read::ErrorType;
-use serde::{Deserialize, Serialize};
+use serde::{
+    de::{Unexpected, Visitor},
+    Deserialize, Deserializer, Serialize,
+};
 use std::{
-    fs,
+    fs::{self, File},
+    io::Read,
    path::{Path, PathBuf},
 };

 /// Structure to allow defining ignored tests, features and files that should
 /// be ignored even when reading.
-#[derive(Debug)]
+#[derive(Debug, Deserialize)]
 struct Ignored {
+    #[serde(default)]
    tests: FxHashSet<Box<str>>,
+    #[serde(default)]
    features: FxHashSet<Box<str>>,
-    files: FxHashSet<Box<str>>,
+    #[serde(default = "TestFlags::empty")]
    flags: TestFlags,
 }

@ -102,16 +110,17 @@ impl Ignored {

    /// Checks if the ignore list contains the given feature name in the list
    /// of features to ignore.
-    pub(crate) fn contains_any_feature(&self, features: &[Box<str>]) -> bool {
-        features
-            .iter()
-            .any(|feature| self.features.contains(feature))
-    }
-
-    /// Checks if the ignore list contains the given file name in the list to
-    /// ignore from reading.
-    pub(crate) fn contains_file(&self, file: &str) -> bool {
-        self.files.contains(file)
+    pub(crate) fn contains_feature(&self, feature: &str) -> bool {
+        if self.features.contains(feature) {
+            return true;
+        }
+        // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`.
+        // This ensures those are also ignored.
+        feature
+            .split('.')
+            .next()
+            .map(|feat| self.features.contains(feat))
+            .unwrap_or_default()
    }

    pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool {
@ -124,74 +133,11 @@ impl Default for Ignored {
        Self {
            tests: FxHashSet::default(),
            features: FxHashSet::default(),
-            files: FxHashSet::default(),
            flags: TestFlags::empty(),
        }
    }
 }

-/// List of ignored tests.
-static IGNORED: Lazy<Ignored> = Lazy::new(|| {
-    let path = Path::new("test_ignore.txt");
-    if path.exists() {
-        let filtered = fs::read_to_string(path).expect("could not read test filters");
-        filtered
-            .lines()
-            .filter(|line| !line.is_empty() && !line.starts_with("//"))
-            .fold(Ignored::default(), |mut ign, line| {
-                // let mut line = line.to_owned();
-                if line.starts_with("file:") {
-                    let file = line
-                        .strip_prefix("file:")
-                        .expect("prefix disappeared")
-                        .trim()
-                        .to_owned()
-                        .into_boxed_str();
-                    let test = if file.ends_with(".js") {
-                        file.strip_suffix(".js")
-                            .expect("suffix disappeared")
-                            .to_owned()
-                            .into_boxed_str()
-                    } else {
-                        file.clone()
-                    };
-                    ign.files.insert(file);
-                    ign.tests.insert(test);
-                } else if line.starts_with("feature:") {
-                    ign.features.insert(
-                        line.strip_prefix("feature:")
-                            .expect("prefix disappeared")
-                            .trim()
-                            .to_owned()
-                            .into_boxed_str(),
-                    );
-                } else if line.starts_with("flag:") {
-                    let flag = line
-                        .strip_prefix("flag:")
-                        .expect("prefix disappeared")
-                        .trim()
-                        .parse::<TestFlag>()
-                        .expect("invalid flag found");
-                    ign.flags.insert(flag.into());
-                } else {
-                    let mut test = line.trim();
-                    if test
-                        .rsplit('.')
-                        .next()
-                        .map(|ext| ext.eq_ignore_ascii_case("js"))
-                        == Some(true)
-                    {
-                        test = test.strip_suffix(".js").expect("suffix disappeared");
-                    }
-                    ign.tests.insert(test.to_owned().into_boxed_str());
-                }
-                ign
-            })
-    } else {
-        Ignored::default()
-    }
-});
-
 /// Boa test262 tester
 #[derive(Debug, Parser)]
 #[command(author, version, about, name = "Boa test262 tester")]
@ -217,6 +163,10 @@ enum Cli {
        /// Execute tests serially
        #[arg(short, long)]
        disable_parallelism: bool,
+
+        /// Path to a TOML file with the ignored tests, features, flags and/or files.
+        #[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)]
+        ignored: PathBuf,
    },
    /// Compare two test suite results.
    Compare {
@ -235,7 +185,8 @@ enum Cli {
 }

 /// Program entry point.
-fn main() {
+fn main() -> Result<()> {
+    color_eyre::install()?;
    match Cli::parse() {
        Cli::Run {
            verbose,
@ -243,23 +194,15 @@ fn main() {
            suite,
            output,
            disable_parallelism,
-        } => {
-            if let Err(e) = run_test_suite(
+            ignored: ignore,
+        } => run_test_suite(
            verbose,
            !disable_parallelism,
            test262_path.as_path(),
            suite.as_path(),
            output.as_deref(),
-            ) {
-                eprintln!("Error: {e}");
-                let mut src = e.source();
-                while let Some(e) = src {
-                    eprintln!("    caused by: {e}");
-                    src = e.source();
-                }
-                std::process::exit(1);
-            }
-        }
+            ignore.as_path(),
+        ),
        Cli::Compare {
            base,
            new,
@ -275,24 +218,33 @@ fn run_test_suite(
    test262_path: &Path,
    suite: &Path,
    output: Option<&Path>,
-) -> anyhow::Result<()> {
+    ignored: &Path,
+) -> Result<()> {
    if let Some(path) = output {
        if path.exists() {
            if !path.is_dir() {
                bail!("the output path must be a directory.");
            }
        } else {
-            fs::create_dir_all(path).context("could not create the output directory")?;
+            fs::create_dir_all(path).wrap_err("could not create the output directory")?;
        }
    }

+    let ignored = {
+        let mut input = String::new();
+        let mut f = File::open(ignored).wrap_err("could not open ignored tests file")?;
+        f.read_to_string(&mut input)
+            .wrap_err("could not read ignored tests file")?;
+        toml::from_str(&input).wrap_err("could not decode ignored tests file")?
+    };
+
    if verbose != 0 {
        println!("Loading the test suite...");
    }
-    let harness = read_harness(test262_path).context("could not read harness")?;
+    let harness = read_harness(test262_path).wrap_err("could not read harness")?;

    if suite.to_string_lossy().ends_with(".js") {
-        let test = read_test(&test262_path.join(suite)).with_context(|| {
+        let test = read_test(&test262_path.join(suite)).wrap_err_with(|| {
            let suite = suite.display();
            format!("could not read the test {suite}")
        })?;
@ -304,7 +256,7 @@ fn run_test_suite(

        println!();
    } else {
-        let suite = read_suite(&test262_path.join(suite)).with_context(|| {
+        let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| {
            let suite = suite.display();
            format!("could not read the suite {suite}")
        })?;
@ -332,7 +284,7 @@ fn run_test_suite(
        );

        write_json(results, output, verbose)
-            .context("could not write the results to the output JSON file")?;
+            .wrap_err("could not write the results to the output JSON file")?;
    }

    Ok(())
@ -419,6 +371,7 @@ struct Test {
    includes: Box<[Box<str>]>,
    locale: Locale,
    content: Box<str>,
+    ignored: bool,
 }

 impl Test {
@ -440,15 +393,12 @@ impl Test {
            includes: metadata.includes,
            locale: metadata.locale,
            content: content.into(),
+            ignored: false,
        }
    }

-    /// Sets the name of the test.
-    fn set_name<N>(&mut self, name: N)
-    where
-        N: Into<Box<str>>,
-    {
-        self.name = name.into();
+    fn set_ignored(&mut self) {
+        self.ignored = true;
    }
 }

@ -534,6 +484,59 @@ where
    }
 }

+impl<'de> Deserialize<'de> for TestFlags {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct FlagsVisitor;
+
+        impl<'de> Visitor<'de> for FlagsVisitor {
+            type Value = TestFlags;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(formatter, "a sequence of flags")
+            }
+
+            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::SeqAccess<'de>,
+            {
+                let mut flags = TestFlags::empty();
+                while let Some(elem) = seq.next_element::<TestFlag>()? {
+                    flags |= elem.into();
+                }
+                Ok(flags)
+            }
+        }
+
+        struct RawFlagsVisitor;
+
+        impl<'de> Visitor<'de> for RawFlagsVisitor {
+            type Value = TestFlags;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(formatter, "a flags number")
+            }
+
+            fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                TestFlags::from_bits(v).ok_or_else(|| {
+                    E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number")
+                })
+            }
+        }
+
+        if deserializer.is_human_readable() {
+            deserializer.deserialize_seq(FlagsVisitor)
+        } else {
+            deserializer.deserialize_u16(RawFlagsVisitor)
+        }
+    }
+}
+
 /// Phase for an error.
 #[derive(Debug, Clone, Copy, Deserialize)]
 #[serde(rename_all = "lowercase")]
--- a/boa_tester/src/read.rs
+++ b/boa_tester/src/read.rs
@ -1,10 +1,15 @@
 //! Module to read the list of test suites from disk.

-use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED};
-use anyhow::Context;
+use crate::Ignored;
+
+use super::{Harness, Locale, Phase, Test, TestSuite};
+use color_eyre::{
+    eyre::{eyre, WrapErr},
+    Result,
+};
 use fxhash::FxHashMap;
 use serde::Deserialize;
-use std::{fs, io, path::Path, str::FromStr};
+use std::{fs, io, path::Path};

 /// Representation of the YAML metadata in Test262 tests.
 #[derive(Debug, Clone, Deserialize)]
@ -77,31 +82,12 @@ pub(super) enum TestFlag {
    NonDeterministic,
 }

-impl FromStr for TestFlag {
-    type Err = String; // TODO: improve error type.
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "onlyStrict" => Ok(Self::OnlyStrict),
-            "noStrict" => Ok(Self::NoStrict),
-            "module" => Ok(Self::Module),
-            "raw" => Ok(Self::Raw),
-            "async" => Ok(Self::Async),
-            "generated" => Ok(Self::Generated),
-            "CanBlockIsFalse" => Ok(Self::CanBlockIsFalse),
-            "CanBlockIsTrue" => Ok(Self::CanBlockIsTrue),
-            "non-deterministic" => Ok(Self::NonDeterministic),
-            _ => Err(format!("unknown test flag: {s}")),
-        }
-    }
-}
-
 /// Reads the Test262 defined bindings.
-pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
+pub(super) fn read_harness(test262_path: &Path) -> Result<Harness> {
    let mut includes = FxHashMap::default();

-    for entry in
-        fs::read_dir(test262_path.join("harness")).context("error reading the harness directory")?
+    for entry in fs::read_dir(test262_path.join("harness"))
+        .wrap_err("error reading the harness directory")?
    {
        let entry = entry?;
        let file_name = entry.file_name();
@ -112,7 +98,7 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
        }

        let content = fs::read_to_string(entry.path())
-            .with_context(|| format!("error reading the harnes/{file_name}"))?;
+            .wrap_err_with(|| format!("error reading the harnes/{file_name}"))?;

        includes.insert(
            file_name.into_owned().into_boxed_str(),
@ -120,13 +106,13 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
        );
    }
    let assert = fs::read_to_string(test262_path.join("harness/assert.js"))
-        .context("error reading harnes/assert.js")?
+        .wrap_err("error reading harnes/assert.js")?
        .into_boxed_str();
    let sta = fs::read_to_string(test262_path.join("harness/sta.js"))
-        .context("error reading harnes/sta.js")?
+        .wrap_err("error reading harnes/sta.js")?
        .into_boxed_str();
    let doneprint_handle = fs::read_to_string(test262_path.join("harness/doneprintHandle.js"))
-        .context("error reading harnes/doneprintHandle.js")?
+        .wrap_err("error reading harnes/doneprintHandle.js")?
        .into_boxed_str();

    Ok(Harness {
@ -138,38 +124,54 @@ pub(super) fn read_harness(test262_path: &Path) -> anyhow::Result<Harness> {
 }

 /// Reads a test suite in the given path.
-pub(super) fn read_suite(path: &Path) -> anyhow::Result<TestSuite> {
+pub(super) fn read_suite(
+    path: &Path,
+    ignored: &Ignored,
+    mut ignore_suite: bool,
+) -> Result<TestSuite> {
    let name = path
        .file_name()
-        .with_context(|| format!("test suite with no name found: {}", path.display()))?
+        .ok_or_else(|| eyre!(format!("test suite with no name found: {}", path.display())))?
        .to_str()
-        .with_context(|| format!("non-UTF-8 suite name found: {}", path.display()))?;
+        .ok_or_else(|| eyre!(format!("non-UTF-8 suite name found: {}", path.display())))?;
+
+    ignore_suite |= ignored.contains_test(name);

    let mut suites = Vec::new();
    let mut tests = Vec::new();

    // TODO: iterate in parallel
-    for entry in path.read_dir().context("retrieving entry")? {
+    for entry in path.read_dir().wrap_err("retrieving entry")? {
        let entry = entry?;

-        if entry.file_type().context("retrieving file type")?.is_dir() {
-            suites.push(read_suite(entry.path().as_path()).with_context(|| {
+        if entry.file_type().wrap_err("retrieving file type")?.is_dir() {
+            suites.push(
+                read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| {
                    let path = entry.path();
                    let suite = path.display();
                    format!("error reading sub-suite {suite}")
-            })?);
+                })?,
+            );
        } else if entry.file_name().to_string_lossy().contains("_FIXTURE") {
            continue;
-        } else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) {
-            let mut test = Test::default();
-            test.set_name(entry.file_name().to_string_lossy());
-            tests.push(test);
        } else {
-            tests.push(read_test(entry.path().as_path()).with_context(|| {
+            let mut test = read_test(entry.path().as_path()).wrap_err_with(|| {
                let path = entry.path();
                let suite = path.display();
                format!("error reading test {suite}")
-            })?);
+            })?;
+
+            if ignore_suite
+                || ignored.contains_any_flag(test.flags)
+                || ignored.contains_test(&test.name)
+                || test
+                    .features
+                    .iter()
+                    .any(|feat| ignored.contains_feature(feat))
+            {
+                test.set_ignored();
+            }
+            tests.push(test);
        }
    }

--- a/boa_tester/src/results.rs
+++ b/boa_tester/src/results.rs
@ -1,4 +1,5 @@
 use super::SuiteResult;
+use color_eyre::{eyre::WrapErr, Result};
 use serde::{Deserialize, Serialize};
 use std::{
    env, fs,
@ -208,16 +209,16 @@ fn update_gh_pages_repo(path: &Path, verbose: u8) {
 }

 /// Compares the results of two test suite runs.
-pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
+pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result<()> {
    let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
-        fs::File::open(base).expect("could not open the base results file"),
+        fs::File::open(base).wrap_err("could not open the base results file")?,
    ))
-    .expect("could not read the base results");
+    .wrap_err("could not read the base results")?;

    let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
-        fs::File::open(new).expect("could not open the new results file"),
+        fs::File::open(new).wrap_err("could not open the new results file")?,
    ))
-    .expect("could not read the new results");
+    .wrap_err("could not read the new results")?;

    let base_total = base_results.results.total as isize;
    let new_total = new_results.results.total as isize;
@ -433,6 +434,8 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
            }
        }
    }
+
+    Ok(())
 }

 /// Test differences.
--- a/test_ignore.toml
+++ b/test_ignore.toml
@ -0,0 +1,45 @@
+# Not implemented yet:
+flags = ["module"]
+
+features = [
+    # Non-implemented features:
+    "json-modules",
+    "SharedArrayBuffer",
+    "resizable-arraybuffer",
+    "Temporal",
+    "tail-call-optimization",
+    "ShadowRealm",
+    "FinalizationRegistry",
+    "Atomics",
+    "dynamic_import",
+    "decorators",
+    "array-grouping",
+    "IsHTMLDDA",
+
+    # Non-implemented Intl features
+    "intl-normative-optional",
+    "Intl.DurationFormat",
+    "Intl.NumberFormat-v3",
+    "Intl.NumberFormat-unified",
+    "Intl.ListFormat",
+    "Intl.DisplayNames",
+    "Intl.RelativeTimeFormat",
+    "Intl.Segmenter",
+    "Intl.Locale",
+
+    # Stage 3 proposals
+
+    # https://github.com/tc39/proposal-symbols-as-weakmap-keys
+    "symbols-as-weakmap-keys",
+
+    # Non-standard
+    "caller",
+
+    # RegExp tests that check individual codepoints.
+    # They are not useful considering the cpu time they waste.
+    "regexp-unicode-property-escapes",
+]
+
+# RegExp tests that check individual codepoints.
+# They are not useful considering the cpu time they waste.
+tests = ["CharacterClassEscapes"]
--- a/test_ignore.txt
+++ b/test_ignore.txt
@ -1,68 +0,0 @@
-// Not implemented yet:
-flag:module
-
-// Non-implemented features:
-feature:json-modules
-feature:SharedArrayBuffer
-feature:resizable-arraybuffer
-feature:Temporal
-feature:tail-call-optimization
-feature:ShadowRealm
-feature:FinalizationRegistry
-feature:FinalizationRegistry.prototype.cleanupSome
-feature:Atomics
-feature:dynamic_import
-feature:decorators
-feature:array-grouping
-feature:IsHTMLDDA
-
-// Non-implemented Intl features
-feature:intl-normative-optional
-feature:Intl.DurationFormat
-feature:Intl.NumberFormat-v3
-feature:Intl.NumberFormat-unified
-feature:Intl.ListFormat
-feature:Intl.DisplayNames
-feature:Intl.RelativeTimeFormat
-feature:Intl.Segmenter
-feature:Intl.Locale
-
-// Non-standard
-feature:caller
-
-// Stage 3 proposals
-
-// https://github.com/tc39/proposal-symbols-as-weakmap-keys
-feature:symbols-as-weakmap-keys
-
-// These generate a stack overflow
-tco-call
-tco-member
-
-// RegExp tests that check individual codepoints.
-// They are not usefull in comparision to the cpu time they waste.
-feature:regexp-unicode-property-escapes
-character-class-non-whitespace-class-escape-plus-quantifier
-character-class-non-whitespace-class-escape-plus-quantifier-flags-u
-character-class-non-word-class-escape-flags-u
-character-class-non-whitespace-class-escape
-character-class-non-digit-class-escape-plus-quantifier-flags-u
-character-class-non-word-class-escape
-character-class-non-whitespace-class-escape-flags-u
-character-class-digit-class-escape-flags-u
-character-class-digit-class-escape
-character-class-non-digit-class-escape-plus-quantifier
-character-class-whitespace-class-escape-flags-u
-character-class-whitespace-class-escape-plus-quantifier-flags-u
-character-class-word-class-escape
-character-class-digit-class-escape-plus-quantifier-flags-u
-character-class-non-digit-class-escape
-character-class-digit-class-escape-plus-quantifier
-character-class-whitespace-class-escape
-character-class-whitespace-class-escape-plus-quantifier
-character-class-word-class-escape-plus-quantifier
-character-class-non-word-class-escape-plus-quantifier
-character-class-word-class-escape-flags-u
-character-class-word-class-escape-plus-quantifier-flags-u
-character-class-non-digit-class-escape-flags-u
-character-class-non-word-class-escape-plus-quantifier-flags-u