Adding VM conformance output to PR checks (#1685)

This PR adds conformance results for the VM branch both for PRs and for the conformance results in GitHub pages (even if these are not currently being shown). I'm not 100% sure how this will really work, as I'm not very used to the syntax to concatenate strings.

Co-authored-by: João Borges <rageknify@gmail.com>
Co-authored-by: RageKnify <RageKnify@gmail.com>

.github/workflows/test262.yml

@@ -51,7 +51,7 @@ jobs:
       # Run the results comparison
       - name: Compare results
         if: github.event_name == 'pull_request'
-        id: compare
+        id: compare-non-vm
         shell: bash
         run: |
           cd boa
@@ -61,6 +61,26 @@ jobs:
           comment="${comment//$'\r'/'%0D'}"
           echo "::set-output name=comment::$comment"
 
+      - name: Run the test262 test suite for VM
+        run: |
+          cd boa
+          mkdir -p ../results
+          cargo run --release --features vm --bin boa_tester -- run -v -o ../results/test262/vm
+          cd ..
+
+      # Run the results comparison for VM
+      - name: Compare results for VM
+        if: github.event_name == 'pull_request'
+        id: compare-vm
+        shell: bash
+        run: |
+          cd boa
+          comment="$(./target/release/boa_tester compare ../gh-pages/test262/refs/heads/main/vm-latest.json ../results/test262/vm/pull/latest.json -m)"
+          comment="${comment//'%'/'%25'}"
+          comment="${comment//$'\n'/'%0A'}"
+          comment="${comment//$'\r'/'%0D'}"
+          echo "::set-output name=comment::$comment"
+
       - name: Get the PR number
         if: github.event_name == 'pull_request'
         id: pr-number
@@ -80,7 +100,11 @@ jobs:
         continue-on-error: true
         with:
           comment-id: ${{ steps.previous-comment.outputs.comment-id }}
-          body: ${{ steps.compare.outputs.comment }}
+          body: |
+            ### Test262 conformance changes
+
+            ${{ steps.compare-non-vm.outputs.comment }}
+            ${{ steps.compare-vm.outputs.comment }}
           edit-mode: replace
 
       - name: Write a new comment
@@ -89,7 +113,11 @@ jobs:
         continue-on-error: true
         with:
           issue-number: ${{ steps.pr-number.outputs.pr }}
-          body: ${{ steps.compare.outputs.comment }}
+          body: |
+            ### Test262 conformance changes
+
+            ${{ steps.compare-non-vm.outputs.comment }}
+            ${{ steps.compare-vm.outputs.comment }}
 
       # Commit changes to GitHub pages.
       - name: Commit files

boa_tester/Cargo.toml

@@ -11,6 +11,9 @@ exclude = ["../.vscode/*", "../Dockerfile", "../Makefile", "../.editorConfig"]
 edition = "2021"
 rust-version = "1.56"
 
+[features]
+vm = ["Boa/vm"]
+
 [dependencies]
 Boa = { path = "../boa" }
 structopt = "0.3.25"

boa_tester/src/results.rs

@@ -202,7 +202,7 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
     let new_conformance = (new_passed as f64 / new_total as f64) * 100_f64;
     let conformance_diff = new_conformance - base_conformance;
 
-    let test_diff = compute_result_diff(base, &base_results.results, &new_results.results);
+    let test_diff = compute_result_diff(Path::new(""), &base_results.results, &new_results.results);
 
     if markdown {
         use num_format::{Locale, ToFormattedString};
@@ -218,7 +218,11 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) {
             )
         }
 
-        println!("### Test262 conformance changes:");
+        #[cfg(feature = "vm")]
+        println!("#### VM implementation");
+        #[cfg(not(feature = "vm"))]
+        println!("#### Non-VM implementation");
+
         println!("| Test result | main count | PR count | difference |");
         println!("| :---------: | :----------: | :------: | :--------: |");
         println!(
@@ -426,9 +430,7 @@ fn compute_result_diff(
         {
             let test_name = format!(
                 "test/{}/{}.js {}(previously {:?})",
-                base.strip_prefix("../gh-pages/test262/refs/heads/main/latest.json")
+                base.display(),
-                    .expect("error removing prefix")
-                    .display(),
                 new_test.name,
                 if base_test.strict {
                     "[strict mode] "