CI workflow improvements (mostly Test262) (#889)

* Improved CI workflows This improves several things in the CI workflows: - More conformant Test262 result generation - Benchmarks should now show comments for all users - Added Test262 result comparison comments to Pull Requests * Fixed typo * Checking the comment generation * Fixing conditions to test comments * Fix a couple of bugs on the comparator * Fixed format * Trying to fix comment updating * Removing commit autor when searching * Replace the comment instead of appending * Switched back to the `pull_request_target` event
4 years ago · be5d198b4d
21 changed files with 1152 additions and 725 deletions
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@ -1,4 +1,4 @@
-on: [pull_request]
+on: [pull_request_target]
 name: Benchmarks
 jobs:
  runBenchmark:
--- a/.github/workflows/test262.yml
+++ b/.github/workflows/test262.yml
@ -5,7 +5,7 @@ on:
      - master
    tags:
      - v*
-  pull_request:
+  pull_request_target:
    branches:
      - master

@ -36,7 +36,6 @@ jobs:

      # Run the test suite and upload the results
      - name: Checkout GitHub pages
-        if: github.event_name == 'push'
        uses: actions/checkout@v2
        with:
          ref: gh-pages
@ -45,16 +44,59 @@ jobs:
      - name: Run the test262 test suite
        run: |
          cd boa
-          cargo run --release --bin boa_tester -- -o ../gh-pages/test262
+          mkdir ../results
+          cargo run --release --bin boa_tester -- run -o ../results/test262
          cd ..

+      # Run the results comparison
+      - name: Compare results
+        if: github.event_name == 'pull_request_target'
+        id: compare
+        shell: bash
+        run: |
+          cd boa
+          comment="$(./target/release/boa_tester compare ../gh-pages/test262/refs/heads/master/latest.json ../results/test262/pull/latest.json -m)"
+          comment="${comment//'%'/'%25'}"
+          comment="${comment//$'\n'/'%0A'}"
+          comment="${comment//$'\r'/'%0D'}"
+          echo "::set-output name=comment::$comment"
+
+      - name: Get the PR number
+        if: github.event_name == 'pull_request_target'
+        id: pr-number
+        uses: kkak10/pr-number-action@v1.3
+
+      - name: Find Previous Comment
+        if: github.event_name == 'pull_request_target'
+        uses: peter-evans/find-comment@v1
+        id: previous-comment
+        with:
+          issue-number: ${{ steps.pr-number.outputs.pr }}
+          body-includes: Test262 conformance changes
+
+      - name: Update comment
+        if: github.event_name == 'pull_request_target' && steps.previous-comment.outputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ steps.previous-comment.outputs.comment-id }}
+          body: ${{ steps.compare.outputs.comment }}
+          edit-mode: replace
+
+      - name: Write a new comment
+        if: github.event_name == 'pull_request_target' && !steps.previous-comment.outputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          issue-number: ${{ steps.pr-number.outputs.pr }}
+          body: ${{ steps.compare.outputs.comment }}
+
+      # Commit changes to GitHub pages.
      - name: Commit files
        if: github.event_name == 'push'
        run: |
+          cp ./results/test262/* ./gh-pages/test262/
          cd gh-pages
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
-          git pull --ff-only
          git add test262
          git commit -m "Add new test262 results" -a
          cd ..
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -72,7 +72,7 @@ suite, you can just run the normal `cargo test`, and to run the full ECMAScript
 with this command:

 ```
-cargo run --release --bin boa_tester -- -v 2> error.log
+cargo run --release --bin boa_tester -- run -v 2> error.log
 ```

 Note that this requires the `test262` submodule to be checked out, so you will need to run the following first:
@ -81,9 +81,15 @@ Note that this requires the `test262` submodule to be checked out, so you will n
 git submodule init && git submodule update
 ```

-This will run the test suite in verbose mode (you can remove the `-- -v` part to run it in non-verbose mode),
+This will run the test suite in verbose mode (you can remove the `-v` part to run it in non-verbose mode),
 and output nice colorings in the terminal. It will also output any panic information into the `error.log` file.

+You can get some more verbose information that tells you the exact name of each test that is being run, useful
+for debugging purposes by setting up the verbose flag twice, for example `-vv`.
+
+Finally, if you want to only run one sub-suite or even one test (to just check if you fixed/broke something specific),
+you can do it with the `-s` parameter, and then passing the path to the sub-suite or test that you want to run.
+
 ## Communication

 We have a Discord server, feel free to ask questions here:
--- a/Cargo.lock
+++ b/Cargo.lock
@ -42,6 +42,15 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "arrayvec"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
+dependencies = [
+ "nodrop",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@ -90,6 +99,7 @@ dependencies = [
 "fxhash",
 "git2",
 "hex",
+ "num-format",
 "once_cell",
 "regex",
 "serde",
@ -108,9 +118,9 @@ dependencies = [

 [[package]]
 name = "bstr"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931"
+checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf"
 dependencies = [
 "lazy_static",
 "memchr",
@ -663,6 +673,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
 [[package]]
 name = "num-bigint"
 version = "0.3.0"
@ -675,6 +691,16 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "num-format"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
+dependencies = [
+ "arrayvec",
+ "itoa",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.43"
@ -1128,9 +1154,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "1.0.44"
+version = "1.0.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd"
+checksum = "ea9c5432ff16d6152371f808fb5a871cd67368171b09bb21b43df8e4a47a3556"
 dependencies = [
 "proc-macro2",
 "quote",
--- a/boa/benches/exec.rs
+++ b/boa/benches/exec.rs
@ -21,7 +21,9 @@ fn symbol_creation(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(SYMBOL_CREATION.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(SYMBOL_CREATION.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("Symbols (Execution)", move |b| {
@ -36,7 +38,7 @@ fn for_loop_execution(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(FOR_LOOP.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(FOR_LOOP.as_bytes(), false).parse_all().unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("For loop (Execution)", move |b| {
@ -51,7 +53,9 @@ fn fibonacci(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(FIBONACCI.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(FIBONACCI.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("Fibonacci (Execution)", move |b| {
@ -66,7 +70,9 @@ fn object_creation(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(OBJECT_CREATION.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(OBJECT_CREATION.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("Object Creation (Execution)", move |b| {
@ -81,7 +87,7 @@ fn object_prop_access_const(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(OBJECT_PROP_ACCESS_CONST.as_bytes())
+    let nodes = Parser::new(OBJECT_PROP_ACCESS_CONST.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -98,7 +104,7 @@ fn object_prop_access_dyn(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(OBJECT_PROP_ACCESS_DYN.as_bytes())
+    let nodes = Parser::new(OBJECT_PROP_ACCESS_DYN.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -115,7 +121,7 @@ fn regexp_literal_creation(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP_LITERAL_CREATION.as_bytes())
+    let nodes = Parser::new(REGEXP_LITERAL_CREATION.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -132,7 +138,9 @@ fn regexp_creation(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP_CREATION.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(REGEXP_CREATION.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("RegExp (Execution)", move |b| {
@ -147,7 +155,9 @@ fn regexp_literal(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP_LITERAL.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(REGEXP_LITERAL.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("RegExp Literal (Execution)", move |b| {
@ -162,7 +172,7 @@ fn regexp(c: &mut Criterion) {
    let mut engine = Context::new();

    // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(REGEXP.as_bytes(), false).parse_all().unwrap();

    // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
    c.bench_function("RegExp (Execution)", move |b| {
@ -175,7 +185,9 @@ static ARRAY_ACCESS: &str = include_str!("bench_scripts/array_access.js");
 fn array_access(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(ARRAY_ACCESS.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(ARRAY_ACCESS.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    c.bench_function("Array access (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@ -187,7 +199,9 @@ static ARRAY_CREATE: &str = include_str!("bench_scripts/array_create.js");
 fn array_creation(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(ARRAY_CREATE.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(ARRAY_CREATE.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    c.bench_function("Array creation (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@ -199,7 +213,9 @@ static ARRAY_POP: &str = include_str!("bench_scripts/array_pop.js");
 fn array_pop(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(ARRAY_POP.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(ARRAY_POP.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    c.bench_function("Array pop (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@ -211,7 +227,9 @@ static STRING_CONCAT: &str = include_str!("bench_scripts/string_concat.js");
 fn string_concat(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(STRING_CONCAT.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(STRING_CONCAT.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    c.bench_function("String concatenation (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@ -223,7 +241,9 @@ static STRING_COMPARE: &str = include_str!("bench_scripts/string_compare.js");
 fn string_compare(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(STRING_COMPARE.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(STRING_COMPARE.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    c.bench_function("String comparison (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@ -235,7 +255,9 @@ static STRING_COPY: &str = include_str!("bench_scripts/string_copy.js");
 fn string_copy(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(STRING_COPY.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(STRING_COPY.as_bytes(), false)
+        .parse_all()
+        .unwrap();

    c.bench_function("String copy (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@ -247,7 +269,7 @@ static NUMBER_OBJECT_ACCESS: &str = include_str!("bench_scripts/number_object_ac
 fn number_object_access(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(NUMBER_OBJECT_ACCESS.as_bytes())
+    let nodes = Parser::new(NUMBER_OBJECT_ACCESS.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -261,7 +283,7 @@ static BOOLEAN_OBJECT_ACCESS: &str = include_str!("bench_scripts/boolean_object_
 fn boolean_object_access(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(BOOLEAN_OBJECT_ACCESS.as_bytes())
+    let nodes = Parser::new(BOOLEAN_OBJECT_ACCESS.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -275,7 +297,7 @@ static STRING_OBJECT_ACCESS: &str = include_str!("bench_scripts/string_object_ac
 fn string_object_access(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(STRING_OBJECT_ACCESS.as_bytes())
+    let nodes = Parser::new(STRING_OBJECT_ACCESS.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -289,7 +311,7 @@ static ARITHMETIC_OPERATIONS: &str = include_str!("bench_scripts/arithmetic_oper
 fn arithmetic_operations(c: &mut Criterion) {
    let mut engine = Context::new();

-    let nodes = Parser::new(ARITHMETIC_OPERATIONS.as_bytes())
+    let nodes = Parser::new(ARITHMETIC_OPERATIONS.as_bytes(), false)
        .parse_all()
        .unwrap();

@ -302,7 +324,7 @@ static CLEAN_JS: &str = include_str!("bench_scripts/clean_js.js");

 fn clean_js(c: &mut Criterion) {
    let mut engine = Context::new();
-    let nodes = Parser::new(CLEAN_JS.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(CLEAN_JS.as_bytes(), false).parse_all().unwrap();
    c.bench_function("Clean js (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
    });
@ -312,7 +334,7 @@ static MINI_JS: &str = include_str!("bench_scripts/mini_js.js");

 fn mini_js(c: &mut Criterion) {
    let mut engine = Context::new();
-    let nodes = Parser::new(MINI_JS.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(MINI_JS.as_bytes(), false).parse_all().unwrap();
    c.bench_function("Mini js (Execution)", move |b| {
        b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
    });
--- a/boa/benches/parser.rs
+++ b/boa/benches/parser.rs
@ -14,7 +14,7 @@ static EXPRESSION: &str = include_str!("bench_scripts/expression.js");

 fn expression_parser(c: &mut Criterion) {
    c.bench_function("Expression (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(EXPRESSION.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(EXPRESSION.as_bytes()), false).parse_all())
    });
 }

@ -22,7 +22,7 @@ static HELLO_WORLD: &str = include_str!("bench_scripts/hello_world.js");

 fn hello_world_parser(c: &mut Criterion) {
    c.bench_function("Hello World (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(HELLO_WORLD.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(HELLO_WORLD.as_bytes()), false).parse_all())
    });
 }

@ -30,7 +30,7 @@ static FOR_LOOP: &str = include_str!("bench_scripts/for_loop.js");

 fn for_loop_parser(c: &mut Criterion) {
    c.bench_function("For loop (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(FOR_LOOP.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(FOR_LOOP.as_bytes()), false).parse_all())
    });
 }

@ -56,7 +56,7 @@ fn long_file_parser(c: &mut Criterion) {

    let file = std::fs::File::open(FILE_NAME).expect("Could not open file");
    c.bench_function("Long file (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(&file)).parse_all())
+        b.iter(|| Parser::new(black_box(&file), false).parse_all())
    });

    fs::remove_file(FILE_NAME).unwrap_or_else(|_| panic!("could not remove {}", FILE_NAME));
@ -66,7 +66,7 @@ static GOAL_SYMBOL_SWITCH: &str = include_str!("bench_scripts/goal_symbol_switch

 fn goal_symbol_switch(c: &mut Criterion) {
    c.bench_function("Goal Symbols (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(GOAL_SYMBOL_SWITCH.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(GOAL_SYMBOL_SWITCH.as_bytes()), false).parse_all())
    });
 }

@ -74,7 +74,7 @@ static CLEAN_JS: &str = include_str!("bench_scripts/clean_js.js");

 fn clean_js(c: &mut Criterion) {
    c.bench_function("Clean js (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(CLEAN_JS.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(CLEAN_JS.as_bytes()), false).parse_all())
    });
 }

@ -82,7 +82,7 @@ static MINI_JS: &str = include_str!("bench_scripts/mini_js.js");

 fn mini_js(c: &mut Criterion) {
    c.bench_function("Mini js (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(MINI_JS.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(MINI_JS.as_bytes()), false).parse_all())
    });
 }

--- a/boa/src/context.rs
+++ b/boa/src/context.rs
@ -241,7 +241,7 @@ impl Context {
        &mut self.executor
    }

-    /// A helper function for getting a immutable reference to the `console` object.
+    /// A helper function for getting an immutable reference to the `console` object.
    #[cfg(feature = "console")]
    pub(crate) fn console(&self) -> &Console {
        &self.console
@ -630,7 +630,7 @@ impl Context {
    pub fn eval(&mut self, src: &str) -> Result<Value> {
        let main_timer = BoaProfiler::global().start_event("Main", "Main");

-        let parsing_result = Parser::new(src.as_bytes())
+        let parsing_result = Parser::new(src.as_bytes(), false)
            .parse_all()
            .map_err(|e| e.to_string());

--- a/boa/src/lib.rs
+++ b/boa/src/lib.rs
@ -78,8 +78,8 @@ pub type Result<T> = StdResult<T, Value>;
 /// It will return either the statement list AST node for the code, or a parsing error if something
 /// goes wrong.
 #[inline]
-pub fn parse(src: &str) -> StdResult<StatementList, ParseError> {
-    Parser::new(src.as_bytes()).parse_all()
+pub fn parse(src: &str, strict_mode: bool) -> StdResult<StatementList, ParseError> {
+    Parser::new(src.as_bytes(), strict_mode).parse_all()
 }

 /// Execute the code using an existing Context
@ -87,7 +87,7 @@ pub fn parse(src: &str) -> StdResult<StatementList, ParseError> {
 #[cfg(test)]
 pub(crate) fn forward(engine: &mut Context, src: &str) -> String {
    // Setup executor
-    let expr = match parse(src) {
+    let expr = match parse(src, false) {
        Ok(res) => res,
        Err(e) => {
            return format!(
@ -114,7 +114,7 @@ pub(crate) fn forward(engine: &mut Context, src: &str) -> String {
 pub(crate) fn forward_val(engine: &mut Context, src: &str) -> Result<Value> {
    let main_timer = BoaProfiler::global().start_event("Main", "Main");
    // Setup executor
-    let result = parse(src)
+    let result = parse(src, false)
        .map_err(|e| {
            engine
                .throw_syntax_error(e.to_string())
--- a/boa/src/syntax/parser/mod.rs
+++ b/boa/src/syntax/parser/mod.rs
@ -88,13 +88,14 @@ pub struct Parser<R> {
 }

 impl<R> Parser<R> {
-    pub fn new(reader: R) -> Self
+    pub fn new(reader: R, strict_mode: bool) -> Self
    where
        R: Read,
    {
-        Self {
-            cursor: Cursor::new(reader),
-        }
+        let mut cursor = Cursor::new(reader);
+        cursor.set_strict_mode(strict_mode);
+
+        Self { cursor }
    }

    pub fn parse_all(&mut self) -> Result<StatementList, ParseError>
--- a/boa/src/syntax/parser/tests.rs
+++ b/boa/src/syntax/parser/tests.rs
@ -19,7 +19,7 @@ where
    L: Into<Box<[Node]>>,
 {
    assert_eq!(
-        Parser::new(js.as_bytes())
+        Parser::new(js.as_bytes(), false)
            .parse_all()
            .expect("failed to parse"),
        StatementList::from(expr)
@ -29,7 +29,7 @@ where
 /// Checks that the given javascript string creates a parse error.
 #[track_caller]
 pub(super) fn check_invalid(js: &str) {
-    assert!(Parser::new(js.as_bytes()).parse_all().is_err());
+    assert!(Parser::new(js.as_bytes(), false).parse_all().is_err());
 }

 /// Should be parsed as `new Class().method()` instead of `new (Class().method())`
--- a/boa/src/value/mod.rs
+++ b/boa/src/value/mod.rs
@ -456,7 +456,7 @@ impl Value {
        let key = key.into();
        match self.get_property(key) {
            Some(ref desc) => match desc {
-                PropertyDescriptor::Accessor(_) => todo!(),
+                PropertyDescriptor::Accessor(_) => todo!("property accessor descriptors"),
                PropertyDescriptor::Data(desc) => desc.value(),
            },
            None => Value::undefined(),
--- a/boa_cli/src/main.rs
+++ b/boa_cli/src/main.rs
@ -107,7 +107,7 @@ arg_enum! {
 fn parse_tokens(src: &str) -> Result<StatementList, String> {
    use boa::syntax::parser::Parser;

-    Parser::new(src.as_bytes())
+    Parser::new(src.as_bytes(), false)
        .parse_all()
        .map_err(|e| format!("ParsingError: {}", e))
 }
--- a/boa_tester/Cargo.toml
+++ b/boa_tester/Cargo.toml
@ -23,3 +23,4 @@ colored = "2.0.0"
 fxhash = "0.2.1"
 git2 = "0.13.12"
 hex = "0.4.2"
+num-format = "0.4.0"
--- a/boa_tester/src/exec.rs
+++ b/boa_tester/src/exec.rs
@ -2,54 +2,48 @@

 use super::{
    Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult,
-    TestSuite, CLI,
+    TestSuite, IGNORED,
 };
-use boa::{parse, Context};
+use boa::{parse, Context, Value};
 use colored::Colorize;
-use fxhash::FxHashSet;
-use once_cell::sync::Lazy;
-use std::{fs, panic, path::Path};
-
-/// List of ignored tests.
-static IGNORED: Lazy<FxHashSet<Box<str>>> = Lazy::new(|| {
-    let path = Path::new("test_ignore.txt");
-    if path.exists() {
-        let filtered = fs::read_to_string(path).expect("could not read test filters");
-        filtered
-            .lines()
-            .filter(|line| !line.is_empty() && !line.starts_with("//"))
-            .map(|line| line.to_owned().into_boxed_str())
-            .collect::<FxHashSet<_>>()
-    } else {
-        FxHashSet::default()
-    }
-});
+use std::panic;

 impl TestSuite {
    /// Runs the test suite.
-    pub(crate) fn run(&self, harness: &Harness) -> SuiteResult {
-        if CLI.verbose() {
+    pub(crate) fn run(&self, harness: &Harness, verbose: u8) -> SuiteResult {
+        if verbose != 0 {
            println!("Suite {}:", self.name);
        }

        // TODO: in parallel
-        let suites: Vec<_> = self.suites.iter().map(|suite| suite.run(harness)).collect();
+        let suites: Vec<_> = self
+            .suites
+            .iter()
+            .map(|suite| suite.run(harness, verbose))
+            .collect();

        // TODO: in parallel
-        let tests: Vec<_> = self.tests.iter().map(|test| test.run(harness)).collect();
+        let tests: Vec<_> = self
+            .tests
+            .iter()
+            .map(|test| test.run(harness, verbose))
+            .flatten()
+            .collect();

-        if CLI.verbose() {
+        if verbose != 0 {
            println!();
        }

        // Count passed tests
        let mut passed = 0;
        let mut ignored = 0;
+        let mut panic = 0;
        for test in &tests {
            match test.result {
                TestOutcomeResult::Passed => passed += 1,
                TestOutcomeResult::Ignored => ignored += 1,
-                _ => {}
+                TestOutcomeResult::Panic => panic += 1,
+                TestOutcomeResult::Failed => {}
            }
        }

@ -59,14 +53,18 @@ impl TestSuite {
            total += suite.total;
            passed += suite.passed;
            ignored += suite.ignored;
+            panic += suite.panic;
        }

-        if CLI.verbose() {
+        if verbose != 0 {
            println!(
-                "Results: total: {}, passed: {}, ignored: {}, conformance: {:.2}%",
+                "Results: total: {}, passed: {}, ignored: {}, failed: {} (panics: {}{}), conformance: {:.2}%",
                total,
-                passed,
-                ignored,
+                passed.to_string().green(),
+                ignored.to_string().yellow(),
+                (total - passed - ignored).to_string().red(),
+                if panic == 0 {"0".normal()} else {panic.to_string().red()},
+                if panic != 0 {" ⚠"} else {""}.red(),
                (passed as f64 / total as f64) * 100.0
            );
        }
@ -76,6 +74,7 @@ impl TestSuite {
            total,
            passed,
            ignored,
+            panic,
            suites,
            tests,
        }
@ -84,79 +83,121 @@ impl TestSuite {

 impl Test {
    /// Runs the test.
-    pub(crate) fn run(&self, harness: &Harness) -> TestResult {
-        // println!("Starting `{}`", self.name);
-
-        let result = if !self.flags.intersects(TestFlags::ASYNC | TestFlags::MODULE)
-            && !IGNORED.contains(&self.name)
-        {
-            let res = panic::catch_unwind(|| {
-                match self.expected_outcome {
-                    Outcome::Positive => {
-                        let mut passed = true;
+    pub(crate) fn run(&self, harness: &Harness, verbose: u8) -> Vec<TestResult> {
+        let mut results = Vec::new();
+        if self.flags.contains(TestFlags::STRICT) {
+            results.push(self.run_once(harness, true, verbose));
+        }

-                        if self.flags.contains(TestFlags::RAW) {
-                            let mut engine = self.set_up_env(&harness, false);
-                            let res = engine.eval(&self.content);
+        if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) {
+            results.push(self.run_once(harness, false, verbose));
+        }

-                            passed = res.is_ok()
-                        } else {
-                            if self.flags.contains(TestFlags::STRICT) {
-                                let mut engine = self.set_up_env(&harness, true);
-                                let res = engine.eval(&self.content);
+        results
+    }

-                                passed = res.is_ok()
+    /// Runs the test once, in strict or non-strict mode
+    fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
+        if verbose > 1 {
+            println!(
+                "Starting `{}`{}",
+                self.name,
+                if strict { " (strict mode)" } else { "" }
+            );
        }

-                            if passed && self.flags.contains(TestFlags::NO_STRICT) {
-                                let mut engine = self.set_up_env(&harness, false);
+        let (result, result_text) = if !IGNORED.contains_any_flag(self.flags)
+            && !IGNORED.contains_test(&self.name)
+            && !IGNORED.contains_any_feature(&self.features)
+            && (matches!(self.expected_outcome, Outcome::Positive)
+                || matches!(self.expected_outcome, Outcome::Negative {
+                    phase: Phase::Parse,
+                    error_type: _,
+                })
+                || matches!(self.expected_outcome, Outcome::Negative {
+                    phase: Phase::Early,
+                    error_type: _,
+                })
+                || matches!(self.expected_outcome, Outcome::Negative {
+                    phase: Phase::Runtime,
+                    error_type: _,
+                })) {
+            let res = panic::catch_unwind(|| match self.expected_outcome {
+                Outcome::Positive => {
+                    // TODO: implement async and add `harness/doneprintHandle.js` to the includes.
+
+                    let mut engine = self.set_up_env(&harness, strict);
                    let res = engine.eval(&self.content);

-                                passed = res.is_ok()
-                            }
-                        }
+                    let passed = res.is_ok();
+                    let text = match res {
+                        Ok(val) => format!("{}", val.display()),
+                        Err(e) => format!("Uncaught {}", e.display()),
+                    };

-                        passed
+                    (passed, text)
                }
                Outcome::Negative {
                    phase: Phase::Parse,
                    ref error_type,
+                }
+                | Outcome::Negative {
+                    phase: Phase::Early,
+                    ref error_type,
                } => {
                    assert_eq!(
                        error_type.as_ref(),
                        "SyntaxError",
-                            "non-SyntaxError parsing error found in {}",
+                        "non-SyntaxError parsing/early error found in {}",
                        self.name
                    );

-                        parse(&self.content).is_err()
+                    match parse(&self.content, strict) {
+                        Ok(n) => (false, format!("{:?}", n)),
+                        Err(e) => (true, format!("Uncaught {}", e)),
+                    }
                }
                Outcome::Negative {
-                        phase: _,
+                    phase: Phase::Resolution,
                    error_type: _,
+                } => todo!("check module resolution errors"),
+                Outcome::Negative {
+                    phase: Phase::Runtime,
+                    ref error_type,
                } => {
-                        // TODO: check the phase
-                        false
+                    if let Err(e) = parse(&self.content, strict) {
+                        (false, format!("Uncaught {}", e))
+                    } else {
+                        let mut engine = self.set_up_env(&harness, strict);
+
+                        match engine.eval(&self.content) {
+                            Ok(res) => (false, format!("{}", res.display())),
+                            Err(e) => {
+                                let passed = e.display().to_string().contains(error_type.as_ref());
+
+                                (passed, format!("Uncaught {}", e.display()))
+                            }
+                        }
                    }
                }
            });

            let result = res
-                .map(|res| {
+                .map(|(res, text)| {
                    if res {
-                        TestOutcomeResult::Passed
+                        (TestOutcomeResult::Passed, text)
                    } else {
-                        TestOutcomeResult::Failed
+                        (TestOutcomeResult::Failed, text)
                    }
                })
                .unwrap_or_else(|_| {
                    eprintln!("last panic was on test \"{}\"", self.name);
-                    TestOutcomeResult::Panic
+                    (TestOutcomeResult::Panic, String::new())
                });

            print!(
                "{}",
-                if let TestOutcomeResult::Passed = result {
+                if let (TestOutcomeResult::Passed, _) = result {
                    ".".green()
                } else {
                    ".".red()
@ -165,15 +206,15 @@ impl Test {

            result
        } else {
-            // Ignoring async tests for now.
-            // TODO: implement async and add `harness/doneprintHandle.js` to the includes.
            print!("{}", ".".yellow());
-            TestOutcomeResult::Ignored
+            (TestOutcomeResult::Ignored, String::new())
        };

        TestResult {
            name: self.name.clone(),
+            strict,
            result,
+            result_text: result_text.into_boxed_str(),
        }
    }

@ -183,7 +224,11 @@ impl Test {
        // TODO: in parallel.
        let mut engine = Context::new();

-        // TODO: set up the environment.
+        // Register the print() function.
+        engine
+            .register_global_function("print", 1, test262_print)
+            .expect("could not register the global print() function");
+        // TODO: add the $262 object.

        if strict {
            engine
@ -212,3 +257,8 @@ impl Test {
        engine
    }
 }
+
+/// `print()` function required by the test262 suite.
+fn test262_print(_this: &Value, _args: &[Value], _context: &mut Context) -> boa::Result<Value> {
+    todo!("print() function");
+}
--- a/boa_tester/src/main.rs
+++ b/boa_tester/src/main.rs
@ -42,10 +42,11 @@ mod results;

 use self::{
    read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
-    results::write_json,
+    results::{compare_results, write_json},
 };
 use bitflags::bitflags;
-use fxhash::FxHashMap;
+use colored::Colorize;
+use fxhash::{FxHashMap, FxHashSet};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 use std::{
@ -54,16 +55,119 @@ use std::{
 };
 use structopt::StructOpt;

-/// CLI information.
-static CLI: Lazy<Cli> = Lazy::new(Cli::from_args);
+/// Structure to allow defining ignored tests, features and files that should
+/// be ignored even when reading.
+#[derive(Debug)]
+struct Ignored {
+    tests: FxHashSet<Box<str>>,
+    features: FxHashSet<Box<str>>,
+    files: FxHashSet<Box<str>>,
+    flags: TestFlags,
+}
+
+impl Ignored {
+    /// Checks if the ignore list contains the given test name in the list of
+    /// tests to ignore.
+    pub(crate) fn contains_test(&self, test: &str) -> bool {
+        self.tests.contains(test)
+    }
+
+    /// Checks if the ignore list contains the given feature name in the list
+    /// of features to ignore.
+    pub(crate) fn contains_any_feature(&self, features: &[Box<str>]) -> bool {
+        features
+            .iter()
+            .any(|feature| self.features.contains(feature))
+    }
+
+    /// Checks if the ignore list contains the given file name in the list to
+    /// ignore from reading.
+    pub(crate) fn contains_file(&self, file: &str) -> bool {
+        self.files.contains(file)
+    }
+
+    pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool {
+        flags.intersects(self.flags)
+    }
+}
+
+impl Default for Ignored {
+    fn default() -> Self {
+        Self {
+            tests: Default::default(),
+            features: Default::default(),
+            files: Default::default(),
+            flags: TestFlags::empty(),
+        }
+    }
+}
+
+/// List of ignored tests.
+static IGNORED: Lazy<Ignored> = Lazy::new(|| {
+    let path = Path::new("test_ignore.txt");
+    if path.exists() {
+        let filtered = fs::read_to_string(path).expect("could not read test filters");
+        filtered
+            .lines()
+            .filter(|line| !line.is_empty() && !line.starts_with("//"))
+            .fold(Ignored::default(), |mut ign, line| {
+                // let mut line = line.to_owned();
+                if line.starts_with("file:") {
+                    let file = line
+                        .strip_prefix("file:")
+                        .expect("prefix disappeared")
+                        .trim()
+                        .to_owned()
+                        .into_boxed_str();
+                    let test = if file.ends_with(".js") {
+                        file.strip_suffix(".js")
+                            .expect("suffix disappeared")
+                            .to_owned()
+                            .into_boxed_str()
+                    } else {
+                        file.clone()
+                    };
+                    ign.files.insert(file);
+                    ign.tests.insert(test);
+                } else if line.starts_with("feature:") {
+                    ign.features.insert(
+                        line.strip_prefix("feature:")
+                            .expect("prefix disappeared")
+                            .trim()
+                            .to_owned()
+                            .into_boxed_str(),
+                    );
+                } else if line.starts_with("flag:") {
+                    let flag = line
+                        .strip_prefix("flag:")
+                        .expect("prefix disappeared")
+                        .trim()
+                        .parse::<TestFlag>()
+                        .expect("invalid flag found");
+                    ign.flags.insert(flag.into())
+                } else {
+                    let mut test = line.trim();
+                    if test.ends_with(".js") {
+                        test = test.strip_suffix(".js").expect("suffix disappeared");
+                    }
+                    ign.tests.insert(test.to_owned().into_boxed_str());
+                }
+                ign
+            })
+    } else {
+        Default::default()
+    }
+});

 /// Boa test262 tester
 #[derive(StructOpt, Debug)]
 #[structopt(name = "Boa test262 tester")]
-struct Cli {
-    // Whether to show verbose output.
-    #[structopt(short, long)]
-    verbose: bool,
+enum Cli {
+    /// Run the test suitr.
+    Run {
+        /// Whether to show verbose output.
+        #[structopt(short, long, parse(from_occurrences))]
+        verbose: u8,

        /// Path to the Test262 suite.
        #[structopt(long, parse(from_os_str), default_value = "./test262")]
@ -76,33 +180,49 @@ struct Cli {
        /// Optional output folder for the full results information.
        #[structopt(short, long, parse(from_os_str))]
        output: Option<PathBuf>,
-}
+    },
+    Compare {
+        /// Base results of the suite.
+        #[structopt(parse(from_os_str))]
+        base: PathBuf,

-impl Cli {
-    // Whether to show verbose output.
-    fn verbose(&self) -> bool {
-        self.verbose
-    }
+        /// New results to compare.
+        #[structopt(parse(from_os_str))]
+        new: PathBuf,

-    /// Path to the Test262 suite.
-    fn test262_path(&self) -> &Path {
-        self.test262_path.as_path()
-    }
+        /// Whether to use markdown output
+        #[structopt(short, long)]
+        markdown: bool,
+    },
+}

-    /// Which specific test or test suite to run.
-    fn suite(&self) -> &Path {
-        self.suite.as_path()
+/// Program entry point.
+fn main() {
+    match Cli::from_args() {
+        Cli::Run {
+            verbose,
+            test262_path,
+            suite,
+            output,
+        } => {
+            run_test_suite(
+                verbose,
+                test262_path.as_path(),
+                suite.as_path(),
+                output.as_deref(),
+            );
        }
-
-    /// Optional output folder for the full results information.
-    fn output(&self) -> Option<&Path> {
-        self.output.as_deref()
+        Cli::Compare {
+            base,
+            new,
+            markdown,
+        } => compare_results(base.as_path(), new.as_path(), markdown),
    }
 }

-/// Program entry point.
-fn main() {
-    if let Some(path) = CLI.output() {
+/// Runs the full test suite.
+fn run_test_suite(verbose: u8, test262_path: &Path, suite: &Path, output: Option<&Path>) {
+    if let Some(path) = output {
        if path.exists() {
            if !path.is_dir() {
                eprintln!("The output path must be a directory.");
@ -113,40 +233,48 @@ fn main() {
        }
    }

-    if CLI.verbose() {
+    if verbose != 0 {
        println!("Loading the test suite...");
    }
-    let harness = read_harness().expect("could not read initialization bindings");
+    let harness = read_harness(test262_path).expect("could not read initialization bindings");

-    if CLI.suite().to_string_lossy().ends_with(".js") {
-        let test = read_test(&CLI.test262_path().join(CLI.suite()))
-            .expect("could not get the test to run");
+    if suite.to_string_lossy().ends_with(".js") {
+        let test = read_test(&test262_path.join(suite)).expect("could not get the test to run");

-        if CLI.verbose() {
+        if verbose != 0 {
            println!("Test loaded, starting...");
        }
-        test.run(&harness);
+        test.run(&harness, verbose);

        println!();
    } else {
-        let suite = read_suite(&CLI.test262_path().join(CLI.suite()))
-            .expect("could not get the list of tests to run");
+        let suite =
+            read_suite(&test262_path.join(suite)).expect("could not get the list of tests to run");

-        if CLI.verbose() {
+        if verbose != 0 {
            println!("Test suite loaded, starting tests...");
        }
-        let results = suite.run(&harness);
+        let results = suite.run(&harness, verbose);

        println!();
        println!("Results:");
        println!("Total tests: {}", results.total);
-        println!("Passed tests: {}", results.passed);
+        println!("Passed tests: {}", results.passed.to_string().green());
+        println!("Ignored tests: {}", results.ignored.to_string().yellow());
+        println!(
+            "Failed tests: {} (panics: {})",
+            (results.total - results.passed - results.ignored)
+                .to_string()
+                .red(),
+            results.panic.to_string().red()
+        );
        println!(
            "Conformance: {:.2}%",
            (results.passed as f64 / results.total as f64) * 100.0
        );

-        write_json(results).expect("could not write the results to the output JSON file");
+        write_json(results, output, verbose)
+            .expect("could not write the results to the output JSON file");
    }
 }

@ -173,15 +301,17 @@ struct SuiteResult {
    name: Box<str>,
    #[serde(rename = "c")]
    total: usize,
-    #[serde(rename = "p")]
+    #[serde(rename = "o")]
    passed: usize,
    #[serde(rename = "i")]
    ignored: usize,
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(rename = "p")]
+    panic: usize,
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    #[serde(rename = "s")]
    suites: Vec<SuiteResult>,
    #[serde(rename = "t")]
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    tests: Vec<TestResult>,
 }

@ -190,6 +320,10 @@ struct SuiteResult {
 struct TestResult {
    #[serde(rename = "n")]
    name: Box<str>,
+    #[serde(rename = "s", default)]
+    strict: bool,
+    #[serde(skip)]
+    result_text: Box<str>,
    #[serde(rename = "r")]
    result: TestOutcomeResult,
 }
@ -207,7 +341,7 @@ enum TestOutcomeResult {
 }

 /// Represents a test.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 struct Test {
    name: Box<str>,
    description: Box<str>,
@ -242,6 +376,14 @@ impl Test {
            content: content.into(),
        }
    }
+
+    /// Sets the name of the test.
+    fn set_name<N>(&mut self, name: N)
+    where
+        N: Into<Box<str>>,
+    {
+        self.name = name.into()
+    }
 }

 /// An outcome for a test.
--- a/boa_tester/src/read.rs
+++ b/boa_tester/src/read.rs
@ -1,9 +1,9 @@
 //! Module to read the list of test suites from disk.

-use super::{Harness, Locale, Phase, Test, TestSuite, CLI};
+use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED};
 use fxhash::FxHashMap;
 use serde::Deserialize;
-use std::{fs, io, path::Path};
+use std::{fs, io, path::Path, str::FromStr};

 /// Representation of the YAML metadata in Test262 tests.
 #[derive(Debug, Clone, Deserialize)]
@ -52,11 +52,30 @@ pub(super) enum TestFlag {
    NonDeterministic,
 }

+impl FromStr for TestFlag {
+    type Err = String; // TODO: improve error type.
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "onlyStrict" => Ok(Self::OnlyStrict),
+            "noStrict" => Ok(Self::NoStrict),
+            "module" => Ok(Self::Module),
+            "raw" => Ok(Self::Raw),
+            "async" => Ok(Self::Async),
+            "generated" => Ok(Self::Generated),
+            "CanBlockIsFalse" => Ok(Self::CanBlockIsFalse),
+            "CanBlockIsTrue" => Ok(Self::CanBlockIsTrue),
+            "non-deterministic" => Ok(Self::NonDeterministic),
+            _ => Err(format!("unknown test flag: {}", s)),
+        }
+    }
+}
+
 /// Reads the Test262 defined bindings.
-pub(super) fn read_harness() -> io::Result<Harness> {
+pub(super) fn read_harness(test262_path: &Path) -> io::Result<Harness> {
    let mut includes = FxHashMap::default();

-    for entry in fs::read_dir(CLI.test262_path().join("harness"))? {
+    for entry in fs::read_dir(test262_path.join("harness"))? {
        let entry = entry?;
        let file_name = entry.file_name();
        let file_name = file_name.to_string_lossy();
@ -72,8 +91,8 @@ pub(super) fn read_harness() -> io::Result<Harness> {
            content.into_boxed_str(),
        );
    }
-    let assert = fs::read_to_string(CLI.test262_path().join("harness/assert.js"))?.into_boxed_str();
-    let sta = fs::read_to_string(CLI.test262_path().join("harness/sta.js"))?.into_boxed_str();
+    let assert = fs::read_to_string(test262_path.join("harness/assert.js"))?.into_boxed_str();
+    let sta = fs::read_to_string(test262_path.join("harness/sta.js"))?.into_boxed_str();

    Ok(Harness {
        assert,
@ -84,8 +103,6 @@ pub(super) fn read_harness() -> io::Result<Harness> {

 /// Reads a test suite in the given path.
 pub(super) fn read_suite(path: &Path) -> io::Result<TestSuite> {
-    use std::ffi::OsStr;
-
    let name = path
        .file_stem()
        .ok_or_else(|| {
@ -105,20 +122,18 @@ pub(super) fn read_suite(path: &Path) -> io::Result<TestSuite> {
    let mut suites = Vec::new();
    let mut tests = Vec::new();

-    let filter = |st: &OsStr| {
-        st.to_string_lossy().ends_with("_FIXTURE.js")
-            // TODO: see if we can fix this.
-            || st.to_string_lossy() == "line-terminator-normalisation-CR.js"
-    };
-
    // TODO: iterate in parallel
    for entry in path.read_dir()? {
        let entry = entry?;

        if entry.file_type()?.is_dir() {
            suites.push(read_suite(entry.path().as_path())?);
-        } else if filter(&entry.file_name()) {
+        } else if entry.file_name().to_string_lossy().ends_with("_FIXTURE.js") {
            continue;
+        } else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) {
+            let mut test = Test::default();
+            test.set_name(entry.file_name().to_string_lossy());
+            tests.push(test)
        } else {
            tests.push(read_test(entry.path().as_path())?);
        }
--- a/boa_tester/src/results.rs
+++ b/boa_tester/src/results.rs
@ -1,4 +1,4 @@
-use super::{SuiteResult, CLI};
+use super::SuiteResult;
 use git2::Repository;
 use hex::ToHex;
 use serde::{Deserialize, Serialize};
@ -28,10 +28,12 @@ struct ReducedResultInfo {
    test262_commit: Box<str>,
    #[serde(rename = "t")]
    total: usize,
-    #[serde(rename = "p")]
+    #[serde(rename = "o")]
    passed: usize,
    #[serde(rename = "i")]
    ignored: usize,
+    #[serde(rename = "p")]
+    panic: usize,
 }

 impl From<ResultInfo> for ReducedResultInfo {
@ -43,6 +45,7 @@ impl From<ResultInfo> for ReducedResultInfo {
            total: info.results.total,
            passed: info.results.passed,
            ignored: info.results.ignored,
+            panic: info.results.panic,
        }
    }
 }
@ -56,13 +59,20 @@ const RESULTS_FILE_NAME: &str = "results.json";
 /// Writes the results of running the test suite to the given JSON output file.
 ///
 /// It will append the results to the ones already present, in an array.
-pub(crate) fn write_json(results: SuiteResult) -> io::Result<()> {
-    if let Some(path) = CLI.output() {
+pub(crate) fn write_json(
+    results: SuiteResult,
+    output: Option<&Path>,
+    verbose: u8,
+) -> io::Result<()> {
+    if let Some(path) = output {
        let mut branch = env::var("GITHUB_REF").unwrap_or_default();
        if branch.starts_with("refs/pull") {
            branch = "pull".to_owned();
        }

+        // We make sure we are using the latest commit information in GitHub pages:
+        update_gh_pages_repo();
+
        let path = if branch.is_empty() {
            path.to_path_buf()
        } else {
@ -71,7 +81,7 @@ pub(crate) fn write_json(results: SuiteResult) -> io::Result<()> {
            folder
        };

-        if CLI.verbose() {
+        if verbose != 0 {
            println!("Writing the results to {}...", path.display());
        }

@ -103,7 +113,7 @@ pub(crate) fn write_json(results: SuiteResult) -> io::Result<()> {
        let output = BufWriter::new(fs::File::create(&all_path)?);
        serde_json::to_writer(output, &all_results)?;

-        if CLI.verbose() {
+        if verbose != 0 {
            println!("Results written correctly");
        }
    }
@ -128,3 +138,149 @@ fn get_test262_commit() -> Box<str> {
        .encode_hex::<String>()
        .into_boxed_str()
 }
+
+/// Updates the GitHub pages repository by pulling latest changes before writing the new things.
+fn update_gh_pages_repo() {
+    if env::var("GITHUB_REF").is_ok() {
+        use std::process::Command;
+
+        // We run the command to pull the gh-pages branch: git -C ../gh-pages/ pull origin
+        Command::new("git")
+            .args(&["-C", "../gh-pages", "pull", "--ff-only"])
+            .output()
+            .expect("could not update GitHub Pages");
+    }
+}
+
+/// Compares the results of two test suite runs.
+pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
+    let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
+        fs::File::open(base).expect("could not open the base results file"),
+    ))
+    .expect("could not read the base results");
+
+    let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
+        fs::File::open(new).expect("could not open the new results file"),
+    ))
+    .expect("could not read the new results");
+
+    let base_total = base_results.results.total as isize;
+    let new_total = new_results.results.total as isize;
+    let total_diff = new_total - base_total;
+
+    let base_passed = base_results.results.passed as isize;
+    let new_passed = new_results.results.passed as isize;
+    let passed_diff = new_passed - base_passed;
+
+    let base_ignored = base_results.results.ignored as isize;
+    let new_ignored = new_results.results.ignored as isize;
+    let ignored_diff = new_ignored - base_ignored;
+
+    let base_failed = base_total - base_passed - base_ignored;
+    let new_failed = new_total - new_passed - new_ignored;
+    let failed_diff = new_failed - base_failed;
+
+    let base_panics = base_results.results.panic as isize;
+    let new_panics = new_results.results.panic as isize;
+    let panic_diff = new_panics - base_panics;
+
+    let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;
+    let new_conformance = (new_passed as f64 / new_total as f64) * 100_f64;
+    let conformance_diff = new_conformance - base_conformance;
+
+    if markdown {
+        use num_format::{Locale, ToFormattedString};
+
+        /// Generates a proper diff format, with some bold text if things change.
+        fn diff_format(diff: isize) -> String {
+            format!(
+                "{}{}{}{}",
+                if diff != 0 { "**" } else { "" },
+                if diff > 0 { "+" } else { "" },
+                diff.to_formatted_string(&Locale::en),
+                if diff != 0 { "**" } else { "" }
+            )
+        }
+
+        println!("### Test262 conformance changes:");
+        println!("| Test result | master count | PR count | difference |");
+        println!("| :---------: | :----------: | :------: | :--------: |");
+        println!(
+            "| Total | {} | {} | {} |",
+            base_total.to_formatted_string(&Locale::en),
+            new_total.to_formatted_string(&Locale::en),
+            diff_format(total_diff),
+        );
+        println!(
+            "| Passed | {} | {} | {} |",
+            base_passed.to_formatted_string(&Locale::en),
+            new_passed.to_formatted_string(&Locale::en),
+            diff_format(passed_diff),
+        );
+        println!(
+            "| Ignored | {} | {} | {} |",
+            base_ignored.to_formatted_string(&Locale::en),
+            new_ignored.to_formatted_string(&Locale::en),
+            diff_format(ignored_diff),
+        );
+        println!(
+            "| Failed | {} | {} | {} |",
+            base_failed.to_formatted_string(&Locale::en),
+            new_failed.to_formatted_string(&Locale::en),
+            diff_format(failed_diff),
+        );
+        println!(
+            "| Panics | {} | {} | {} |",
+            base_panics.to_formatted_string(&Locale::en),
+            new_panics.to_formatted_string(&Locale::en),
+            diff_format(panic_diff),
+        );
+        println!(
+            "| Conformance | {:.2} | {:.2} | {} |",
+            base_conformance,
+            new_conformance,
+            format!(
+                "{}{}{:.2}%{}",
+                if conformance_diff.abs() > f64::EPSILON {
+                    "**"
+                } else {
+                    ""
+                },
+                if conformance_diff > 0_f64 { "+" } else { "" },
+                conformance_diff,
+                if conformance_diff.abs() > f64::EPSILON {
+                    "**"
+                } else {
+                    ""
+                },
+            ),
+        );
+    } else {
+        println!("Test262 conformance changes:");
+        println!("| Test result | master |    PR   | difference |");
+        println!(
+            "|    Passed   | {:^6} | {:^5} | {:^10} |",
+            base_passed,
+            new_passed,
+            base_passed - new_passed
+        );
+        println!(
+            "|   Ignored   | {:^6} | {:^5} | {:^10} |",
+            base_ignored,
+            new_ignored,
+            base_ignored - new_ignored
+        );
+        println!(
+            "|   Failed    | {:^6} | {:^5} | {:^10} |",
+            base_failed,
+            new_failed,
+            base_failed - new_failed,
+        );
+        println!(
+            "|   Panics    | {:^6} | {:^5} | {:^10} |",
+            base_panics,
+            new_panics,
+            base_panics - new_panics
+        );
+    }
+}
--- a/boa_wasm/src/lib.rs
+++ b/boa_wasm/src/lib.rs
@ -6,7 +6,7 @@ pub fn evaluate(src: &str) -> Result<String, JsValue> {
    // Setup executor
    let mut engine = Context::new();

-    let expr = match parse(src) {
+    let expr = match parse(src, false) {
        Ok(res) => res,
        Err(e) => {
            return Err(format!(
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 323905b70e644d90faa957235f8ac59eac4ba8ba
+Subproject commit 3439564fcac38845669c8488d68f3d16965a7852
--- a/test_ignore.txt
+++ b/test_ignore.txt
@ -1,28 +1,34 @@
+// This causes issues when loading:
+file:line-terminator-normalisation-CR.js
+
+// Not implemented yet:
+flag:module
+flag:async
+
+// Non-implemented features:
+feature:TypedArray
+//feature:generators
+//feature:async-iteration
+//feature:class
+
 // This does not break the tester but it does iterate from 0 to u32::MAX,
 // because of incorect implementation of `Array.prototype.indexOf`.
 // TODO: Fix it do iterate on the elements in the array **in insertion order**, not from
 // 0 to u32::MAX untill it reaches the element.
 15.4.4.14-5-13

-// New errors:
-// Stack overflows:
-tco-non-eval-function
-tco-non-eval-global
-value-tojson-array-circular
-value-array-circular
-value-tojson-object-circular
-value-object-circular
-
-// This does not stack overflow, but freezes the computer:
-arg-length-exceeding-integer-limit
-
 // These seem to run forever:
-15.4.4.22-9-b-9
-15.4.4.22-7-11
-15.4.4.22-9-5
-15.4.4.22-8-b-iii-1-30
-15.4.4.22-10-3
+arg-length-exceeding-integer-limit
 15.4.4.19-8-c-ii-1
 fill-string-empty
 S15.4.4.10_A3_T2
 S15.4.4.10_A3_T1
+15.4.4.15-3-9
+15.4.4.15-3-28
+length-near-integer-limit
+15.4.4.15-5-12
+15.4.4.15-3-7
+15.4.4.15-3-25
+15.4.4.15-8-9
+length-boundaries
+throws-if-integer-limit-exceeded
--- a/yarn.lock
+++ b/yarn.lock