CI workflow improvements (mostly Test262) (#889)

* Improved CI workflows

This improves several things in the CI workflows:

 - More conformant Test262 result generation
 - Benchmarks should now show comments for all users
 - Added Test262 result comparison comments to Pull Requests

* Fixed typo

* Checking the comment generation

* Fixing conditions to test comments

* Fix a couple of bugs on the comparator

* Fixed format

* Trying to fix comment updating

* Removing commit autor when searching

* Replace the comment instead of appending

* Switched back to the `pull_request_target` event

.github/workflows/pull_request.yml

@@ -1,4 +1,4 @@
-on: [pull_request]
+on: [pull_request_target]
 name: Benchmarks
 jobs:
   runBenchmark:

.github/workflows/test262.yml

@@ -5,7 +5,7 @@ on:
       - master
     tags:
       - v*
-  pull_request:
+  pull_request_target:
     branches:
       - master
 
@@ -36,7 +36,6 @@ jobs:
 
       # Run the test suite and upload the results
       - name: Checkout GitHub pages
-        if: github.event_name == 'push'
         uses: actions/checkout@v2
         with:
           ref: gh-pages
@@ -45,16 +44,59 @@ jobs:
       - name: Run the test262 test suite
         run: |
           cd boa
-          cargo run --release --bin boa_tester -- -o ../gh-pages/test262
+          mkdir ../results
+          cargo run --release --bin boa_tester -- run -o ../results/test262
           cd ..
 
+      # Run the results comparison
+      - name: Compare results
+        if: github.event_name == 'pull_request_target'
+        id: compare
+        shell: bash
+        run: |
+          cd boa
+          comment="$(./target/release/boa_tester compare ../gh-pages/test262/refs/heads/master/latest.json ../results/test262/pull/latest.json -m)"
+          comment="${comment//'%'/'%25'}"
+          comment="${comment//$'\n'/'%0A'}"
+          comment="${comment//$'\r'/'%0D'}"
+          echo "::set-output name=comment::$comment"
+
+      - name: Get the PR number
+        if: github.event_name == 'pull_request_target'
+        id: pr-number
+        uses: kkak10/pr-number-action@v1.3
+
+      - name: Find Previous Comment
+        if: github.event_name == 'pull_request_target'
+        uses: peter-evans/find-comment@v1
+        id: previous-comment
+        with:
+          issue-number: ${{ steps.pr-number.outputs.pr }}
+          body-includes: Test262 conformance changes
+
+      - name: Update comment
+        if: github.event_name == 'pull_request_target' && steps.previous-comment.outputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ steps.previous-comment.outputs.comment-id }}
+          body: ${{ steps.compare.outputs.comment }}
+          edit-mode: replace
+
+      - name: Write a new comment
+        if: github.event_name == 'pull_request_target' && !steps.previous-comment.outputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          issue-number: ${{ steps.pr-number.outputs.pr }}
+          body: ${{ steps.compare.outputs.comment }}
+
+      # Commit changes to GitHub pages.
       - name: Commit files
         if: github.event_name == 'push'
         run: |
+          cp ./results/test262/* ./gh-pages/test262/
           cd gh-pages
           git config --local user.email "action@github.com"
           git config --local user.name "GitHub Action"
-          git pull --ff-only
           git add test262
           git commit -m "Add new test262 results" -a
           cd ..

CONTRIBUTING.md

@@ -72,7 +72,7 @@ suite, you can just run the normal `cargo test`, and to run the full ECMAScript
 with this command:
 
 ```
-cargo run --release --bin boa_tester -- -v 2> error.log
+cargo run --release --bin boa_tester -- run -v 2> error.log
 ```
 
 Note that this requires the `test262` submodule to be checked out, so you will need to run the following first:
@@ -81,9 +81,15 @@ Note that this requires the `test262` submodule to be checked out, so you will n
 git submodule init && git submodule update
 ```
 
-This will run the test suite in verbose mode (you can remove the `-- -v` part to run it in non-verbose mode),
+This will run the test suite in verbose mode (you can remove the `-v` part to run it in non-verbose mode),
 and output nice colorings in the terminal. It will also output any panic information into the `error.log` file.
 
+You can get some more verbose information that tells you the exact name of each test that is being run, useful
+for debugging purposes by setting up the verbose flag twice, for example `-vv`.
+
+Finally, if you want to only run one sub-suite or even one test (to just check if you fixed/broke something specific),
+you can do it with the `-s` parameter, and then passing the path to the sub-suite or test that you want to run.
+
 ## Communication
 
 We have a Discord server, feel free to ask questions here:

Cargo.lock

@@ -42,6 +42,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "arrayvec"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
+dependencies = [
+ "nodrop",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -90,6 +99,7 @@ dependencies = [
  "fxhash",
  "git2",
  "hex",
+ "num-format",
  "once_cell",
  "regex",
  "serde",
@@ -108,9 +118,9 @@ dependencies = [
 
 [[package]]
 name = "bstr"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931"
+checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf"
 dependencies = [
  "lazy_static",
  "memchr",
@@ -663,6 +673,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
 [[package]]
 name = "num-bigint"
 version = "0.3.0"
@@ -675,6 +691,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "num-format"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
+dependencies = [
+ "arrayvec",
+ "itoa",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.43"
@@ -1128,9 +1154,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.44"
+version = "1.0.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd"
+checksum = "ea9c5432ff16d6152371f808fb5a871cd67368171b09bb21b43df8e4a47a3556"
 dependencies = [
  "proc-macro2",
  "quote",

boa/benches/exec.rs

@@ -21,7 +21,9 @@ fn symbol_creation(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(SYMBOL_CREATION.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(SYMBOL_CREATION.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("Symbols (Execution)", move |b| {
@@ -36,7 +38,7 @@ fn for_loop_execution(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(FOR_LOOP.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(FOR_LOOP.as_bytes(), false).parse_all().unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("For loop (Execution)", move |b| {
@@ -51,7 +53,9 @@ fn fibonacci(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(FIBONACCI.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(FIBONACCI.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("Fibonacci (Execution)", move |b| {
@@ -66,7 +70,9 @@ fn object_creation(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(OBJECT_CREATION.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(OBJECT_CREATION.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("Object Creation (Execution)", move |b| {
@@ -81,7 +87,7 @@ fn object_prop_access_const(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(OBJECT_PROP_ACCESS_CONST.as_bytes())
+    let nodes = Parser::new(OBJECT_PROP_ACCESS_CONST.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -98,7 +104,7 @@ fn object_prop_access_dyn(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(OBJECT_PROP_ACCESS_DYN.as_bytes())
+    let nodes = Parser::new(OBJECT_PROP_ACCESS_DYN.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -115,7 +121,7 @@ fn regexp_literal_creation(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP_LITERAL_CREATION.as_bytes())
+    let nodes = Parser::new(REGEXP_LITERAL_CREATION.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -132,7 +138,9 @@ fn regexp_creation(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP_CREATION.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(REGEXP_CREATION.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("RegExp (Execution)", move |b| {
@@ -147,7 +155,9 @@ fn regexp_literal(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP_LITERAL.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(REGEXP_LITERAL.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("RegExp Literal (Execution)", move |b| {
@@ -162,7 +172,7 @@ fn regexp(c: &mut Criterion) {
     let mut engine = Context::new();
 
     // Parse the AST nodes.
-    let nodes = Parser::new(REGEXP.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(REGEXP.as_bytes(), false).parse_all().unwrap();
 
     // Execute the parsed nodes, passing them through a black box, to avoid over-optimizing by the compiler
     c.bench_function("RegExp (Execution)", move |b| {
@@ -175,7 +185,9 @@ static ARRAY_ACCESS: &str = include_str!("bench_scripts/array_access.js");
 fn array_access(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(ARRAY_ACCESS.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(ARRAY_ACCESS.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     c.bench_function("Array access (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@@ -187,7 +199,9 @@ static ARRAY_CREATE: &str = include_str!("bench_scripts/array_create.js");
 fn array_creation(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(ARRAY_CREATE.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(ARRAY_CREATE.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     c.bench_function("Array creation (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@@ -199,7 +213,9 @@ static ARRAY_POP: &str = include_str!("bench_scripts/array_pop.js");
 fn array_pop(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(ARRAY_POP.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(ARRAY_POP.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     c.bench_function("Array pop (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@@ -211,7 +227,9 @@ static STRING_CONCAT: &str = include_str!("bench_scripts/string_concat.js");
 fn string_concat(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(STRING_CONCAT.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(STRING_CONCAT.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     c.bench_function("String concatenation (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@@ -223,7 +241,9 @@ static STRING_COMPARE: &str = include_str!("bench_scripts/string_compare.js");
 fn string_compare(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(STRING_COMPARE.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(STRING_COMPARE.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     c.bench_function("String comparison (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@@ -235,7 +255,9 @@ static STRING_COPY: &str = include_str!("bench_scripts/string_copy.js");
 fn string_copy(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(STRING_COPY.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(STRING_COPY.as_bytes(), false)
+        .parse_all()
+        .unwrap();
 
     c.bench_function("String copy (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
@@ -247,7 +269,7 @@ static NUMBER_OBJECT_ACCESS: &str = include_str!("bench_scripts/number_object_ac
 fn number_object_access(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(NUMBER_OBJECT_ACCESS.as_bytes())
+    let nodes = Parser::new(NUMBER_OBJECT_ACCESS.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -261,7 +283,7 @@ static BOOLEAN_OBJECT_ACCESS: &str = include_str!("bench_scripts/boolean_object_
 fn boolean_object_access(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(BOOLEAN_OBJECT_ACCESS.as_bytes())
+    let nodes = Parser::new(BOOLEAN_OBJECT_ACCESS.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -275,7 +297,7 @@ static STRING_OBJECT_ACCESS: &str = include_str!("bench_scripts/string_object_ac
 fn string_object_access(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(STRING_OBJECT_ACCESS.as_bytes())
+    let nodes = Parser::new(STRING_OBJECT_ACCESS.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -289,7 +311,7 @@ static ARITHMETIC_OPERATIONS: &str = include_str!("bench_scripts/arithmetic_oper
 fn arithmetic_operations(c: &mut Criterion) {
     let mut engine = Context::new();
 
-    let nodes = Parser::new(ARITHMETIC_OPERATIONS.as_bytes())
+    let nodes = Parser::new(ARITHMETIC_OPERATIONS.as_bytes(), false)
         .parse_all()
         .unwrap();
 
@@ -302,7 +324,7 @@ static CLEAN_JS: &str = include_str!("bench_scripts/clean_js.js");
 
 fn clean_js(c: &mut Criterion) {
     let mut engine = Context::new();
-    let nodes = Parser::new(CLEAN_JS.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(CLEAN_JS.as_bytes(), false).parse_all().unwrap();
     c.bench_function("Clean js (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
     });
@@ -312,7 +334,7 @@ static MINI_JS: &str = include_str!("bench_scripts/mini_js.js");
 
 fn mini_js(c: &mut Criterion) {
     let mut engine = Context::new();
-    let nodes = Parser::new(MINI_JS.as_bytes()).parse_all().unwrap();
+    let nodes = Parser::new(MINI_JS.as_bytes(), false).parse_all().unwrap();
     c.bench_function("Mini js (Execution)", move |b| {
         b.iter(|| black_box(&nodes).run(&mut engine).unwrap())
     });

boa/benches/parser.rs

@@ -14,7 +14,7 @@ static EXPRESSION: &str = include_str!("bench_scripts/expression.js");
 
 fn expression_parser(c: &mut Criterion) {
     c.bench_function("Expression (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(EXPRESSION.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(EXPRESSION.as_bytes()), false).parse_all())
     });
 }
 
@@ -22,7 +22,7 @@ static HELLO_WORLD: &str = include_str!("bench_scripts/hello_world.js");
 
 fn hello_world_parser(c: &mut Criterion) {
     c.bench_function("Hello World (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(HELLO_WORLD.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(HELLO_WORLD.as_bytes()), false).parse_all())
     });
 }
 
@@ -30,7 +30,7 @@ static FOR_LOOP: &str = include_str!("bench_scripts/for_loop.js");
 
 fn for_loop_parser(c: &mut Criterion) {
     c.bench_function("For loop (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(FOR_LOOP.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(FOR_LOOP.as_bytes()), false).parse_all())
     });
 }
 
@@ -56,7 +56,7 @@ fn long_file_parser(c: &mut Criterion) {
 
     let file = std::fs::File::open(FILE_NAME).expect("Could not open file");
     c.bench_function("Long file (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(&file)).parse_all())
+        b.iter(|| Parser::new(black_box(&file), false).parse_all())
     });
 
     fs::remove_file(FILE_NAME).unwrap_or_else(|_| panic!("could not remove {}", FILE_NAME));
@@ -66,7 +66,7 @@ static GOAL_SYMBOL_SWITCH: &str = include_str!("bench_scripts/goal_symbol_switch
 
 fn goal_symbol_switch(c: &mut Criterion) {
     c.bench_function("Goal Symbols (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(GOAL_SYMBOL_SWITCH.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(GOAL_SYMBOL_SWITCH.as_bytes()), false).parse_all())
     });
 }
 
@@ -74,7 +74,7 @@ static CLEAN_JS: &str = include_str!("bench_scripts/clean_js.js");
 
 fn clean_js(c: &mut Criterion) {
     c.bench_function("Clean js (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(CLEAN_JS.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(CLEAN_JS.as_bytes()), false).parse_all())
     });
 }
 
@@ -82,7 +82,7 @@ static MINI_JS: &str = include_str!("bench_scripts/mini_js.js");
 
 fn mini_js(c: &mut Criterion) {
     c.bench_function("Mini js (Parser)", move |b| {
-        b.iter(|| Parser::new(black_box(MINI_JS.as_bytes())).parse_all())
+        b.iter(|| Parser::new(black_box(MINI_JS.as_bytes()), false).parse_all())
     });
 }
 

boa/src/context.rs

@@ -241,7 +241,7 @@ impl Context {
         &mut self.executor
     }
 
-    /// A helper function for getting a immutable reference to the `console` object.
+    /// A helper function for getting an immutable reference to the `console` object.
     #[cfg(feature = "console")]
     pub(crate) fn console(&self) -> &Console {
         &self.console
@@ -630,7 +630,7 @@ impl Context {
     pub fn eval(&mut self, src: &str) -> Result<Value> {
         let main_timer = BoaProfiler::global().start_event("Main", "Main");
 
-        let parsing_result = Parser::new(src.as_bytes())
+        let parsing_result = Parser::new(src.as_bytes(), false)
             .parse_all()
             .map_err(|e| e.to_string());
 

boa/src/lib.rs

@@ -78,8 +78,8 @@ pub type Result<T> = StdResult<T, Value>;
 /// It will return either the statement list AST node for the code, or a parsing error if something
 /// goes wrong.
 #[inline]
-pub fn parse(src: &str) -> StdResult<StatementList, ParseError> {
+pub fn parse(src: &str, strict_mode: bool) -> StdResult<StatementList, ParseError> {
-    Parser::new(src.as_bytes()).parse_all()
+    Parser::new(src.as_bytes(), strict_mode).parse_all()
 }
 
 /// Execute the code using an existing Context
@@ -87,7 +87,7 @@ pub fn parse(src: &str) -> StdResult<StatementList, ParseError> {
 #[cfg(test)]
 pub(crate) fn forward(engine: &mut Context, src: &str) -> String {
     // Setup executor
-    let expr = match parse(src) {
+    let expr = match parse(src, false) {
         Ok(res) => res,
         Err(e) => {
             return format!(
@@ -114,7 +114,7 @@ pub(crate) fn forward(engine: &mut Context, src: &str) -> String {
 pub(crate) fn forward_val(engine: &mut Context, src: &str) -> Result<Value> {
     let main_timer = BoaProfiler::global().start_event("Main", "Main");
     // Setup executor
-    let result = parse(src)
+    let result = parse(src, false)
         .map_err(|e| {
             engine
                 .throw_syntax_error(e.to_string())

boa/src/syntax/parser/mod.rs

@@ -88,13 +88,14 @@ pub struct Parser<R> {
 }
 
 impl<R> Parser<R> {
-    pub fn new(reader: R) -> Self
+    pub fn new(reader: R, strict_mode: bool) -> Self
     where
         R: Read,
     {
-        Self {
+        let mut cursor = Cursor::new(reader);
-            cursor: Cursor::new(reader),
+        cursor.set_strict_mode(strict_mode);
-        }
+
+        Self { cursor }
     }
 
     pub fn parse_all(&mut self) -> Result<StatementList, ParseError>

boa/src/syntax/parser/tests.rs

@@ -19,7 +19,7 @@ where
     L: Into<Box<[Node]>>,
 {
     assert_eq!(
-        Parser::new(js.as_bytes())
+        Parser::new(js.as_bytes(), false)
             .parse_all()
             .expect("failed to parse"),
         StatementList::from(expr)
@@ -29,7 +29,7 @@ where
 /// Checks that the given javascript string creates a parse error.
 #[track_caller]
 pub(super) fn check_invalid(js: &str) {
-    assert!(Parser::new(js.as_bytes()).parse_all().is_err());
+    assert!(Parser::new(js.as_bytes(), false).parse_all().is_err());
 }
 
 /// Should be parsed as `new Class().method()` instead of `new (Class().method())`

boa/src/value/mod.rs

@@ -456,7 +456,7 @@ impl Value {
         let key = key.into();
         match self.get_property(key) {
             Some(ref desc) => match desc {
-                PropertyDescriptor::Accessor(_) => todo!(),
+                PropertyDescriptor::Accessor(_) => todo!("property accessor descriptors"),
                 PropertyDescriptor::Data(desc) => desc.value(),
             },
             None => Value::undefined(),

boa_cli/src/main.rs

@@ -107,7 +107,7 @@ arg_enum! {
 fn parse_tokens(src: &str) -> Result<StatementList, String> {
     use boa::syntax::parser::Parser;
 
-    Parser::new(src.as_bytes())
+    Parser::new(src.as_bytes(), false)
         .parse_all()
         .map_err(|e| format!("ParsingError: {}", e))
 }

boa_tester/Cargo.toml

@@ -23,3 +23,4 @@ colored = "2.0.0"
 fxhash = "0.2.1"
 git2 = "0.13.12"
 hex = "0.4.2"
+num-format = "0.4.0"

boa_tester/src/exec.rs

@@ -2,54 +2,48 @@
 
 use super::{
     Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult,
-    TestSuite, CLI,
+    TestSuite, IGNORED,
 };
-use boa::{parse, Context};
+use boa::{parse, Context, Value};
 use colored::Colorize;
-use fxhash::FxHashSet;
+use std::panic;
-use once_cell::sync::Lazy;
-use std::{fs, panic, path::Path};
-
-/// List of ignored tests.
-static IGNORED: Lazy<FxHashSet<Box<str>>> = Lazy::new(|| {
-    let path = Path::new("test_ignore.txt");
-    if path.exists() {
-        let filtered = fs::read_to_string(path).expect("could not read test filters");
-        filtered
-            .lines()
-            .filter(|line| !line.is_empty() && !line.starts_with("//"))
-            .map(|line| line.to_owned().into_boxed_str())
-            .collect::<FxHashSet<_>>()
-    } else {
-        FxHashSet::default()
-    }
-});
 
 impl TestSuite {
     /// Runs the test suite.
-    pub(crate) fn run(&self, harness: &Harness) -> SuiteResult {
+    pub(crate) fn run(&self, harness: &Harness, verbose: u8) -> SuiteResult {
-        if CLI.verbose() {
+        if verbose != 0 {
             println!("Suite {}:", self.name);
         }
 
         // TODO: in parallel
-        let suites: Vec<_> = self.suites.iter().map(|suite| suite.run(harness)).collect();
+        let suites: Vec<_> = self
+            .suites
+            .iter()
+            .map(|suite| suite.run(harness, verbose))
+            .collect();
 
         // TODO: in parallel
-        let tests: Vec<_> = self.tests.iter().map(|test| test.run(harness)).collect();
+        let tests: Vec<_> = self
+            .tests
+            .iter()
+            .map(|test| test.run(harness, verbose))
+            .flatten()
+            .collect();
 
-        if CLI.verbose() {
+        if verbose != 0 {
             println!();
         }
 
         // Count passed tests
         let mut passed = 0;
         let mut ignored = 0;
+        let mut panic = 0;
         for test in &tests {
             match test.result {
                 TestOutcomeResult::Passed => passed += 1,
                 TestOutcomeResult::Ignored => ignored += 1,
-                _ => {}
+                TestOutcomeResult::Panic => panic += 1,
+                TestOutcomeResult::Failed => {}
             }
         }
 
@@ -59,14 +53,18 @@ impl TestSuite {
             total += suite.total;
             passed += suite.passed;
             ignored += suite.ignored;
+            panic += suite.panic;
         }
 
-        if CLI.verbose() {
+        if verbose != 0 {
             println!(
-                "Results: total: {}, passed: {}, ignored: {}, conformance: {:.2}%",
+                "Results: total: {}, passed: {}, ignored: {}, failed: {} (panics: {}{}), conformance: {:.2}%",
                 total,
-                passed,
+                passed.to_string().green(),
-                ignored,
+                ignored.to_string().yellow(),
+                (total - passed - ignored).to_string().red(),
+                if panic == 0 {"0".normal()} else {panic.to_string().red()},
+                if panic != 0 {" ⚠"} else {""}.red(),
                 (passed as f64 / total as f64) * 100.0
             );
         }
@@ -76,6 +74,7 @@ impl TestSuite {
             total,
             passed,
             ignored,
+            panic,
             suites,
             tests,
         }
@@ -84,79 +83,121 @@ impl TestSuite {
 
 impl Test {
     /// Runs the test.
-    pub(crate) fn run(&self, harness: &Harness) -> TestResult {
+    pub(crate) fn run(&self, harness: &Harness, verbose: u8) -> Vec<TestResult> {
-        // println!("Starting `{}`", self.name);
+        let mut results = Vec::new();
-
+        if self.flags.contains(TestFlags::STRICT) {
-        let result = if !self.flags.intersects(TestFlags::ASYNC | TestFlags::MODULE)
+            results.push(self.run_once(harness, true, verbose));
-            && !IGNORED.contains(&self.name)
+        }
-        {
-            let res = panic::catch_unwind(|| {
-                match self.expected_outcome {
-                    Outcome::Positive => {
-                        let mut passed = true;
-
-                        if self.flags.contains(TestFlags::RAW) {
-                            let mut engine = self.set_up_env(&harness, false);
-                            let res = engine.eval(&self.content);
-
-                            passed = res.is_ok()
-                        } else {
-                            if self.flags.contains(TestFlags::STRICT) {
-                                let mut engine = self.set_up_env(&harness, true);
-                                let res = engine.eval(&self.content);
-
-                                passed = res.is_ok()
-                            }
 
-                            if passed && self.flags.contains(TestFlags::NO_STRICT) {
+        if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) {
-                                let mut engine = self.set_up_env(&harness, false);
+            results.push(self.run_once(harness, false, verbose));
-                                let res = engine.eval(&self.content);
+        }
 
-                                passed = res.is_ok()
+        results
-                            }
+    }
-                        }
 
-                        passed
+    /// Runs the test once, in strict or non-strict mode
-                    }
+    fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
-                    Outcome::Negative {
+        if verbose > 1 {
-                        phase: Phase::Parse,
+            println!(
-                        ref error_type,
+                "Starting `{}`{}",
-                    } => {
+                self.name,
-                        assert_eq!(
+                if strict { " (strict mode)" } else { "" }
-                            error_type.as_ref(),
+            );
-                            "SyntaxError",
+        }
-                            "non-SyntaxError parsing error found in {}",
+
-                            self.name
+        let (result, result_text) = if !IGNORED.contains_any_flag(self.flags)
-                        );
+            && !IGNORED.contains_test(&self.name)
-
+            && !IGNORED.contains_any_feature(&self.features)
-                        parse(&self.content).is_err()
+            && (matches!(self.expected_outcome, Outcome::Positive)
+                || matches!(self.expected_outcome, Outcome::Negative {
+                    phase: Phase::Parse,
+                    error_type: _,
+                })
+                || matches!(self.expected_outcome, Outcome::Negative {
+                    phase: Phase::Early,
+                    error_type: _,
+                })
+                || matches!(self.expected_outcome, Outcome::Negative {
+                    phase: Phase::Runtime,
+                    error_type: _,
+                })) {
+            let res = panic::catch_unwind(|| match self.expected_outcome {
+                Outcome::Positive => {
+                    // TODO: implement async and add `harness/doneprintHandle.js` to the includes.
+
+                    let mut engine = self.set_up_env(&harness, strict);
+                    let res = engine.eval(&self.content);
+
+                    let passed = res.is_ok();
+                    let text = match res {
+                        Ok(val) => format!("{}", val.display()),
+                        Err(e) => format!("Uncaught {}", e.display()),
+                    };
+
+                    (passed, text)
+                }
+                Outcome::Negative {
+                    phase: Phase::Parse,
+                    ref error_type,
+                }
+                | Outcome::Negative {
+                    phase: Phase::Early,
+                    ref error_type,
+                } => {
+                    assert_eq!(
+                        error_type.as_ref(),
+                        "SyntaxError",
+                        "non-SyntaxError parsing/early error found in {}",
+                        self.name
+                    );
+
+                    match parse(&self.content, strict) {
+                        Ok(n) => (false, format!("{:?}", n)),
+                        Err(e) => (true, format!("Uncaught {}", e)),
                     }
-                    Outcome::Negative {
+                }
-                        phase: _,
+                Outcome::Negative {
-                        error_type: _,
+                    phase: Phase::Resolution,
-                    } => {
+                    error_type: _,
-                        // TODO: check the phase
+                } => todo!("check module resolution errors"),
-                        false
+                Outcome::Negative {
+                    phase: Phase::Runtime,
+                    ref error_type,
+                } => {
+                    if let Err(e) = parse(&self.content, strict) {
+                        (false, format!("Uncaught {}", e))
+                    } else {
+                        let mut engine = self.set_up_env(&harness, strict);
+
+                        match engine.eval(&self.content) {
+                            Ok(res) => (false, format!("{}", res.display())),
+                            Err(e) => {
+                                let passed = e.display().to_string().contains(error_type.as_ref());
+
+                                (passed, format!("Uncaught {}", e.display()))
+                            }
+                        }
                     }
                 }
             });
 
             let result = res
-                .map(|res| {
+                .map(|(res, text)| {
                     if res {
-                        TestOutcomeResult::Passed
+                        (TestOutcomeResult::Passed, text)
                     } else {
-                        TestOutcomeResult::Failed
+                        (TestOutcomeResult::Failed, text)
                     }
                 })
                 .unwrap_or_else(|_| {
                     eprintln!("last panic was on test \"{}\"", self.name);
-                    TestOutcomeResult::Panic
+                    (TestOutcomeResult::Panic, String::new())
                 });
 
             print!(
                 "{}",
-                if let TestOutcomeResult::Passed = result {
+                if let (TestOutcomeResult::Passed, _) = result {
                     ".".green()
                 } else {
                     ".".red()
@@ -165,15 +206,15 @@ impl Test {
 
             result
         } else {
-            // Ignoring async tests for now.
-            // TODO: implement async and add `harness/doneprintHandle.js` to the includes.
             print!("{}", ".".yellow());
-            TestOutcomeResult::Ignored
+            (TestOutcomeResult::Ignored, String::new())
         };
 
         TestResult {
             name: self.name.clone(),
+            strict,
             result,
+            result_text: result_text.into_boxed_str(),
         }
     }
 
@@ -183,7 +224,11 @@ impl Test {
         // TODO: in parallel.
         let mut engine = Context::new();
 
-        // TODO: set up the environment.
+        // Register the print() function.
+        engine
+            .register_global_function("print", 1, test262_print)
+            .expect("could not register the global print() function");
+        // TODO: add the $262 object.
 
         if strict {
             engine
@@ -212,3 +257,8 @@ impl Test {
         engine
     }
 }
+
+/// `print()` function required by the test262 suite.
+fn test262_print(_this: &Value, _args: &[Value], _context: &mut Context) -> boa::Result<Value> {
+    todo!("print() function");
+}

boa_tester/src/main.rs

@@ -42,10 +42,11 @@ mod results;
 
 use self::{
     read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
-    results::write_json,
+    results::{compare_results, write_json},
 };
 use bitflags::bitflags;
-use fxhash::FxHashMap;
+use colored::Colorize;
+use fxhash::{FxHashMap, FxHashSet};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -54,55 +55,174 @@ use std::{
 };
 use structopt::StructOpt;
 
-/// CLI information.
+/// Structure to allow defining ignored tests, features and files that should
-static CLI: Lazy<Cli> = Lazy::new(Cli::from_args);
+/// be ignored even when reading.
-
+#[derive(Debug)]
-/// Boa test262 tester
+struct Ignored {
-#[derive(StructOpt, Debug)]
+    tests: FxHashSet<Box<str>>,
-#[structopt(name = "Boa test262 tester")]
+    features: FxHashSet<Box<str>>,
-struct Cli {
+    files: FxHashSet<Box<str>>,
-    // Whether to show verbose output.
+    flags: TestFlags,
-    #[structopt(short, long)]
-    verbose: bool,
-
-    /// Path to the Test262 suite.
-    #[structopt(long, parse(from_os_str), default_value = "./test262")]
-    test262_path: PathBuf,
-
-    /// Which specific test or test suite to run.
-    #[structopt(short, long, parse(from_os_str), default_value = "test")]
-    suite: PathBuf,
-
-    /// Optional output folder for the full results information.
-    #[structopt(short, long, parse(from_os_str))]
-    output: Option<PathBuf>,
 }
 
-impl Cli {
+impl Ignored {
-    // Whether to show verbose output.
+    /// Checks if the ignore list contains the given test name in the list of
-    fn verbose(&self) -> bool {
+    /// tests to ignore.
-        self.verbose
+    pub(crate) fn contains_test(&self, test: &str) -> bool {
+        self.tests.contains(test)
     }
 
-    /// Path to the Test262 suite.
+    /// Checks if the ignore list contains the given feature name in the list
-    fn test262_path(&self) -> &Path {
+    /// of features to ignore.
-        self.test262_path.as_path()
+    pub(crate) fn contains_any_feature(&self, features: &[Box<str>]) -> bool {
+        features
+            .iter()
+            .any(|feature| self.features.contains(feature))
     }
 
-    /// Which specific test or test suite to run.
+    /// Checks if the ignore list contains the given file name in the list to
-    fn suite(&self) -> &Path {
+    /// ignore from reading.
-        self.suite.as_path()
+    pub(crate) fn contains_file(&self, file: &str) -> bool {
+        self.files.contains(file)
     }
 
-    /// Optional output folder for the full results information.
+    pub(crate) fn contains_any_flag(&self, flags: TestFlags) -> bool {
-    fn output(&self) -> Option<&Path> {
+        flags.intersects(self.flags)
-        self.output.as_deref()
     }
 }
 
+impl Default for Ignored {
+    fn default() -> Self {
+        Self {
+            tests: Default::default(),
+            features: Default::default(),
+            files: Default::default(),
+            flags: TestFlags::empty(),
+        }
+    }
+}
+
+/// List of ignored tests.
+static IGNORED: Lazy<Ignored> = Lazy::new(|| {
+    let path = Path::new("test_ignore.txt");
+    if path.exists() {
+        let filtered = fs::read_to_string(path).expect("could not read test filters");
+        filtered
+            .lines()
+            .filter(|line| !line.is_empty() && !line.starts_with("//"))
+            .fold(Ignored::default(), |mut ign, line| {
+                // let mut line = line.to_owned();
+                if line.starts_with("file:") {
+                    let file = line
+                        .strip_prefix("file:")
+                        .expect("prefix disappeared")
+                        .trim()
+                        .to_owned()
+                        .into_boxed_str();
+                    let test = if file.ends_with(".js") {
+                        file.strip_suffix(".js")
+                            .expect("suffix disappeared")
+                            .to_owned()
+                            .into_boxed_str()
+                    } else {
+                        file.clone()
+                    };
+                    ign.files.insert(file);
+                    ign.tests.insert(test);
+                } else if line.starts_with("feature:") {
+                    ign.features.insert(
+                        line.strip_prefix("feature:")
+                            .expect("prefix disappeared")
+                            .trim()
+                            .to_owned()
+                            .into_boxed_str(),
+                    );
+                } else if line.starts_with("flag:") {
+                    let flag = line
+                        .strip_prefix("flag:")
+                        .expect("prefix disappeared")
+                        .trim()
+                        .parse::<TestFlag>()
+                        .expect("invalid flag found");
+                    ign.flags.insert(flag.into())
+                } else {
+                    let mut test = line.trim();
+                    if test.ends_with(".js") {
+                        test = test.strip_suffix(".js").expect("suffix disappeared");
+                    }
+                    ign.tests.insert(test.to_owned().into_boxed_str());
+                }
+                ign
+            })
+    } else {
+        Default::default()
+    }
+});
+
+/// Boa test262 tester
+#[derive(StructOpt, Debug)]
+#[structopt(name = "Boa test262 tester")]
+enum Cli {
+    /// Run the test suitr.
+    Run {
+        /// Whether to show verbose output.
+        #[structopt(short, long, parse(from_occurrences))]
+        verbose: u8,
+
+        /// Path to the Test262 suite.
+        #[structopt(long, parse(from_os_str), default_value = "./test262")]
+        test262_path: PathBuf,
+
+        /// Which specific test or test suite to run.
+        #[structopt(short, long, parse(from_os_str), default_value = "test")]
+        suite: PathBuf,
+
+        /// Optional output folder for the full results information.
+        #[structopt(short, long, parse(from_os_str))]
+        output: Option<PathBuf>,
+    },
+    Compare {
+        /// Base results of the suite.
+        #[structopt(parse(from_os_str))]
+        base: PathBuf,
+
+        /// New results to compare.
+        #[structopt(parse(from_os_str))]
+        new: PathBuf,
+
+        /// Whether to use markdown output
+        #[structopt(short, long)]
+        markdown: bool,
+    },
+}
+
 /// Program entry point.
 fn main() {
-    if let Some(path) = CLI.output() {
+    match Cli::from_args() {
+        Cli::Run {
+            verbose,
+            test262_path,
+            suite,
+            output,
+        } => {
+            run_test_suite(
+                verbose,
+                test262_path.as_path(),
+                suite.as_path(),
+                output.as_deref(),
+            );
+        }
+        Cli::Compare {
+            base,
+            new,
+            markdown,
+        } => compare_results(base.as_path(), new.as_path(), markdown),
+    }
+}
+
+/// Runs the full test suite.
+fn run_test_suite(verbose: u8, test262_path: &Path, suite: &Path, output: Option<&Path>) {
+    if let Some(path) = output {
         if path.exists() {
             if !path.is_dir() {
                 eprintln!("The output path must be a directory.");
@@ -113,40 +233,48 @@ fn main() {
         }
     }
 
-    if CLI.verbose() {
+    if verbose != 0 {
         println!("Loading the test suite...");
     }
-    let harness = read_harness().expect("could not read initialization bindings");
+    let harness = read_harness(test262_path).expect("could not read initialization bindings");
 
-    if CLI.suite().to_string_lossy().ends_with(".js") {
+    if suite.to_string_lossy().ends_with(".js") {
-        let test = read_test(&CLI.test262_path().join(CLI.suite()))
+        let test = read_test(&test262_path.join(suite)).expect("could not get the test to run");
-            .expect("could not get the test to run");
 
-        if CLI.verbose() {
+        if verbose != 0 {
             println!("Test loaded, starting...");
         }
-        test.run(&harness);
+        test.run(&harness, verbose);
 
         println!();
     } else {
-        let suite = read_suite(&CLI.test262_path().join(CLI.suite()))
+        let suite =
-            .expect("could not get the list of tests to run");
+            read_suite(&test262_path.join(suite)).expect("could not get the list of tests to run");
 
-        if CLI.verbose() {
+        if verbose != 0 {
             println!("Test suite loaded, starting tests...");
         }
-        let results = suite.run(&harness);
+        let results = suite.run(&harness, verbose);
 
         println!();
         println!("Results:");
         println!("Total tests: {}", results.total);
-        println!("Passed tests: {}", results.passed);
+        println!("Passed tests: {}", results.passed.to_string().green());
+        println!("Ignored tests: {}", results.ignored.to_string().yellow());
+        println!(
+            "Failed tests: {} (panics: {})",
+            (results.total - results.passed - results.ignored)
+                .to_string()
+                .red(),
+            results.panic.to_string().red()
+        );
         println!(
             "Conformance: {:.2}%",
             (results.passed as f64 / results.total as f64) * 100.0
         );
 
-        write_json(results).expect("could not write the results to the output JSON file");
+        write_json(results, output, verbose)
+            .expect("could not write the results to the output JSON file");
     }
 }
 
@@ -173,15 +301,17 @@ struct SuiteResult {
     name: Box<str>,
     #[serde(rename = "c")]
     total: usize,
-    #[serde(rename = "p")]
+    #[serde(rename = "o")]
     passed: usize,
     #[serde(rename = "i")]
     ignored: usize,
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(rename = "p")]
+    panic: usize,
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
     #[serde(rename = "s")]
     suites: Vec<SuiteResult>,
     #[serde(rename = "t")]
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
     tests: Vec<TestResult>,
 }
 
@@ -190,6 +320,10 @@ struct SuiteResult {
 struct TestResult {
     #[serde(rename = "n")]
     name: Box<str>,
+    #[serde(rename = "s", default)]
+    strict: bool,
+    #[serde(skip)]
+    result_text: Box<str>,
     #[serde(rename = "r")]
     result: TestOutcomeResult,
 }
@@ -207,7 +341,7 @@ enum TestOutcomeResult {
 }
 
 /// Represents a test.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 struct Test {
     name: Box<str>,
     description: Box<str>,
@@ -242,6 +376,14 @@ impl Test {
             content: content.into(),
         }
     }
+
+    /// Sets the name of the test.
+    fn set_name<N>(&mut self, name: N)
+    where
+        N: Into<Box<str>>,
+    {
+        self.name = name.into()
+    }
 }
 
 /// An outcome for a test.

boa_tester/src/read.rs

@@ -1,9 +1,9 @@
 //! Module to read the list of test suites from disk.
 
-use super::{Harness, Locale, Phase, Test, TestSuite, CLI};
+use super::{Harness, Locale, Phase, Test, TestSuite, IGNORED};
 use fxhash::FxHashMap;
 use serde::Deserialize;
-use std::{fs, io, path::Path};
+use std::{fs, io, path::Path, str::FromStr};
 
 /// Representation of the YAML metadata in Test262 tests.
 #[derive(Debug, Clone, Deserialize)]
@@ -52,11 +52,30 @@ pub(super) enum TestFlag {
     NonDeterministic,
 }
 
+impl FromStr for TestFlag {
+    type Err = String; // TODO: improve error type.
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "onlyStrict" => Ok(Self::OnlyStrict),
+            "noStrict" => Ok(Self::NoStrict),
+            "module" => Ok(Self::Module),
+            "raw" => Ok(Self::Raw),
+            "async" => Ok(Self::Async),
+            "generated" => Ok(Self::Generated),
+            "CanBlockIsFalse" => Ok(Self::CanBlockIsFalse),
+            "CanBlockIsTrue" => Ok(Self::CanBlockIsTrue),
+            "non-deterministic" => Ok(Self::NonDeterministic),
+            _ => Err(format!("unknown test flag: {}", s)),
+        }
+    }
+}
+
 /// Reads the Test262 defined bindings.
-pub(super) fn read_harness() -> io::Result<Harness> {
+pub(super) fn read_harness(test262_path: &Path) -> io::Result<Harness> {
     let mut includes = FxHashMap::default();
 
-    for entry in fs::read_dir(CLI.test262_path().join("harness"))? {
+    for entry in fs::read_dir(test262_path.join("harness"))? {
         let entry = entry?;
         let file_name = entry.file_name();
         let file_name = file_name.to_string_lossy();
@@ -72,8 +91,8 @@ pub(super) fn read_harness() -> io::Result<Harness> {
             content.into_boxed_str(),
         );
     }
-    let assert = fs::read_to_string(CLI.test262_path().join("harness/assert.js"))?.into_boxed_str();
+    let assert = fs::read_to_string(test262_path.join("harness/assert.js"))?.into_boxed_str();
-    let sta = fs::read_to_string(CLI.test262_path().join("harness/sta.js"))?.into_boxed_str();
+    let sta = fs::read_to_string(test262_path.join("harness/sta.js"))?.into_boxed_str();
 
     Ok(Harness {
         assert,
@@ -84,8 +103,6 @@ pub(super) fn read_harness() -> io::Result<Harness> {
 
 /// Reads a test suite in the given path.
 pub(super) fn read_suite(path: &Path) -> io::Result<TestSuite> {
-    use std::ffi::OsStr;
-
     let name = path
         .file_stem()
         .ok_or_else(|| {
@@ -105,20 +122,18 @@ pub(super) fn read_suite(path: &Path) -> io::Result<TestSuite> {
     let mut suites = Vec::new();
     let mut tests = Vec::new();
 
-    let filter = |st: &OsStr| {
-        st.to_string_lossy().ends_with("_FIXTURE.js")
-            // TODO: see if we can fix this.
-            || st.to_string_lossy() == "line-terminator-normalisation-CR.js"
-    };
-
     // TODO: iterate in parallel
     for entry in path.read_dir()? {
         let entry = entry?;
 
         if entry.file_type()?.is_dir() {
             suites.push(read_suite(entry.path().as_path())?);
-        } else if filter(&entry.file_name()) {
+        } else if entry.file_name().to_string_lossy().ends_with("_FIXTURE.js") {
             continue;
+        } else if IGNORED.contains_file(&entry.file_name().to_string_lossy()) {
+            let mut test = Test::default();
+            test.set_name(entry.file_name().to_string_lossy());
+            tests.push(test)
         } else {
             tests.push(read_test(entry.path().as_path())?);
         }

boa_tester/src/results.rs

@@ -1,4 +1,4 @@
-use super::{SuiteResult, CLI};
+use super::SuiteResult;
 use git2::Repository;
 use hex::ToHex;
 use serde::{Deserialize, Serialize};
@@ -28,10 +28,12 @@ struct ReducedResultInfo {
     test262_commit: Box<str>,
     #[serde(rename = "t")]
     total: usize,
-    #[serde(rename = "p")]
+    #[serde(rename = "o")]
     passed: usize,
     #[serde(rename = "i")]
     ignored: usize,
+    #[serde(rename = "p")]
+    panic: usize,
 }
 
 impl From<ResultInfo> for ReducedResultInfo {
@@ -43,6 +45,7 @@ impl From<ResultInfo> for ReducedResultInfo {
             total: info.results.total,
             passed: info.results.passed,
             ignored: info.results.ignored,
+            panic: info.results.panic,
         }
     }
 }
@@ -56,13 +59,20 @@ const RESULTS_FILE_NAME: &str = "results.json";
 /// Writes the results of running the test suite to the given JSON output file.
 ///
 /// It will append the results to the ones already present, in an array.
-pub(crate) fn write_json(results: SuiteResult) -> io::Result<()> {
+pub(crate) fn write_json(
-    if let Some(path) = CLI.output() {
+    results: SuiteResult,
+    output: Option<&Path>,
+    verbose: u8,
+) -> io::Result<()> {
+    if let Some(path) = output {
         let mut branch = env::var("GITHUB_REF").unwrap_or_default();
         if branch.starts_with("refs/pull") {
             branch = "pull".to_owned();
         }
 
+        // We make sure we are using the latest commit information in GitHub pages:
+        update_gh_pages_repo();
+
         let path = if branch.is_empty() {
             path.to_path_buf()
         } else {
@@ -71,7 +81,7 @@ pub(crate) fn write_json(results: SuiteResult) -> io::Result<()> {
             folder
         };
 
-        if CLI.verbose() {
+        if verbose != 0 {
             println!("Writing the results to {}...", path.display());
         }
 
@@ -103,7 +113,7 @@ pub(crate) fn write_json(results: SuiteResult) -> io::Result<()> {
         let output = BufWriter::new(fs::File::create(&all_path)?);
         serde_json::to_writer(output, &all_results)?;
 
-        if CLI.verbose() {
+        if verbose != 0 {
             println!("Results written correctly");
         }
     }
@@ -128,3 +138,149 @@ fn get_test262_commit() -> Box<str> {
         .encode_hex::<String>()
         .into_boxed_str()
 }
+
+/// Updates the GitHub pages repository by pulling latest changes before writing the new things.
+fn update_gh_pages_repo() {
+    if env::var("GITHUB_REF").is_ok() {
+        use std::process::Command;
+
+        // We run the command to pull the gh-pages branch: git -C ../gh-pages/ pull origin
+        Command::new("git")
+            .args(&["-C", "../gh-pages", "pull", "--ff-only"])
+            .output()
+            .expect("could not update GitHub Pages");
+    }
+}
+
+/// Compares the results of two test suite runs.
+pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
+    let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
+        fs::File::open(base).expect("could not open the base results file"),
+    ))
+    .expect("could not read the base results");
+
+    let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
+        fs::File::open(new).expect("could not open the new results file"),
+    ))
+    .expect("could not read the new results");
+
+    let base_total = base_results.results.total as isize;
+    let new_total = new_results.results.total as isize;
+    let total_diff = new_total - base_total;
+
+    let base_passed = base_results.results.passed as isize;
+    let new_passed = new_results.results.passed as isize;
+    let passed_diff = new_passed - base_passed;
+
+    let base_ignored = base_results.results.ignored as isize;
+    let new_ignored = new_results.results.ignored as isize;
+    let ignored_diff = new_ignored - base_ignored;
+
+    let base_failed = base_total - base_passed - base_ignored;
+    let new_failed = new_total - new_passed - new_ignored;
+    let failed_diff = new_failed - base_failed;
+
+    let base_panics = base_results.results.panic as isize;
+    let new_panics = new_results.results.panic as isize;
+    let panic_diff = new_panics - base_panics;
+
+    let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;
+    let new_conformance = (new_passed as f64 / new_total as f64) * 100_f64;
+    let conformance_diff = new_conformance - base_conformance;
+
+    if markdown {
+        use num_format::{Locale, ToFormattedString};
+
+        /// Generates a proper diff format, with some bold text if things change.
+        fn diff_format(diff: isize) -> String {
+            format!(
+                "{}{}{}{}",
+                if diff != 0 { "**" } else { "" },
+                if diff > 0 { "+" } else { "" },
+                diff.to_formatted_string(&Locale::en),
+                if diff != 0 { "**" } else { "" }
+            )
+        }
+
+        println!("### Test262 conformance changes:");
+        println!("| Test result | master count | PR count | difference |");
+        println!("| :---------: | :----------: | :------: | :--------: |");
+        println!(
+            "| Total | {} | {} | {} |",
+            base_total.to_formatted_string(&Locale::en),
+            new_total.to_formatted_string(&Locale::en),
+            diff_format(total_diff),
+        );
+        println!(
+            "| Passed | {} | {} | {} |",
+            base_passed.to_formatted_string(&Locale::en),
+            new_passed.to_formatted_string(&Locale::en),
+            diff_format(passed_diff),
+        );
+        println!(
+            "| Ignored | {} | {} | {} |",
+            base_ignored.to_formatted_string(&Locale::en),
+            new_ignored.to_formatted_string(&Locale::en),
+            diff_format(ignored_diff),
+        );
+        println!(
+            "| Failed | {} | {} | {} |",
+            base_failed.to_formatted_string(&Locale::en),
+            new_failed.to_formatted_string(&Locale::en),
+            diff_format(failed_diff),
+        );
+        println!(
+            "| Panics | {} | {} | {} |",
+            base_panics.to_formatted_string(&Locale::en),
+            new_panics.to_formatted_string(&Locale::en),
+            diff_format(panic_diff),
+        );
+        println!(
+            "| Conformance | {:.2} | {:.2} | {} |",
+            base_conformance,
+            new_conformance,
+            format!(
+                "{}{}{:.2}%{}",
+                if conformance_diff.abs() > f64::EPSILON {
+                    "**"
+                } else {
+                    ""
+                },
+                if conformance_diff > 0_f64 { "+" } else { "" },
+                conformance_diff,
+                if conformance_diff.abs() > f64::EPSILON {
+                    "**"
+                } else {
+                    ""
+                },
+            ),
+        );
+    } else {
+        println!("Test262 conformance changes:");
+        println!("| Test result | master |    PR   | difference |");
+        println!(
+            "|    Passed   | {:^6} | {:^5} | {:^10} |",
+            base_passed,
+            new_passed,
+            base_passed - new_passed
+        );
+        println!(
+            "|   Ignored   | {:^6} | {:^5} | {:^10} |",
+            base_ignored,
+            new_ignored,
+            base_ignored - new_ignored
+        );
+        println!(
+            "|   Failed    | {:^6} | {:^5} | {:^10} |",
+            base_failed,
+            new_failed,
+            base_failed - new_failed,
+        );
+        println!(
+            "|   Panics    | {:^6} | {:^5} | {:^10} |",
+            base_panics,
+            new_panics,
+            base_panics - new_panics
+        );
+    }
+}

boa_wasm/src/lib.rs

@@ -6,7 +6,7 @@ pub fn evaluate(src: &str) -> Result<String, JsValue> {
     // Setup executor
     let mut engine = Context::new();
 
-    let expr = match parse(src) {
+    let expr = match parse(src, false) {
         Ok(res) => res,
         Err(e) => {
             return Err(format!(

test262

@@ -1 +1 @@
-Subproject commit 323905b70e644d90faa957235f8ac59eac4ba8ba
+Subproject commit 3439564fcac38845669c8488d68f3d16965a7852

test_ignore.txt

@@ -1,28 +1,34 @@
+// This causes issues when loading:
+file:line-terminator-normalisation-CR.js
+
+// Not implemented yet:
+flag:module
+flag:async
+
+// Non-implemented features:
+feature:TypedArray
+//feature:generators
+//feature:async-iteration
+//feature:class
+
 // This does not break the tester but it does iterate from 0 to u32::MAX,
 // because of incorect implementation of `Array.prototype.indexOf`.
 // TODO: Fix it do iterate on the elements in the array **in insertion order**, not from
 // 0 to u32::MAX untill it reaches the element.
 15.4.4.14-5-13
 
-// New errors:
-// Stack overflows:
-tco-non-eval-function
-tco-non-eval-global
-value-tojson-array-circular
-value-array-circular
-value-tojson-object-circular
-value-object-circular
-
-// This does not stack overflow, but freezes the computer:
-arg-length-exceeding-integer-limit
-
 // These seem to run forever:
-15.4.4.22-9-b-9
+arg-length-exceeding-integer-limit
-15.4.4.22-7-11
-15.4.4.22-9-5
-15.4.4.22-8-b-iii-1-30
-15.4.4.22-10-3
 15.4.4.19-8-c-ii-1
 fill-string-empty
 S15.4.4.10_A3_T2
 S15.4.4.10_A3_T1
+15.4.4.15-3-9
+15.4.4.15-3-28
+length-near-integer-limit
+15.4.4.15-5-12
+15.4.4.15-3-7
+15.4.4.15-3-25
+15.4.4.15-8-9
+length-boundaries
+throws-if-integer-limit-exceeded