From 02f737aac7ecc8828a44cc1c65f5ed9f1d643709 Mon Sep 17 00:00:00 2001 From: Zack Mitkin Date: Mon, 20 Mar 2023 22:31:10 +0000 Subject: [PATCH] Add ES5 and ES6 Conformance calculation to boa_tester (#2690) This is calculated based the tests `es5id` or `es6id`. Judging by [this](https://github.com/tc39/test262/issues/1557) it's probably not the best method of doing it, but the alternative would require a lot of rework on the boa_tester so that it can pull an older version of the test262 spec which has it's own problems since there's not really an "ES5" only version. I would think that if we're 100% passing on es5id's then it's safe to assume boa supports es5 (assuming test262's es5id is covering all the es5 cases) This Pull Request fixes/closes #2629. It changes the following: - Store `spec_version` in TestResult based on the tests `es[6/5]id` - Count all es5, es6 and their test outcome during `TestSuite::run()` - Print the conformance. I'm serializing the `spec_version` outcomes so that it can be displayed in test262 github page. I'd like to work on that too if possible. Let me know if there's anything more I should cover in this PR, I'll gladly do it :) Co-authored-by: jedel1043 --- boa_tester/src/exec/mod.rs | 79 +++++++++++++++++++++++------------ boa_tester/src/main.rs | 85 +++++++++++++++++++++++++++++++------- boa_tester/src/results.rs | 39 +++++++++-------- 3 files changed, 142 insertions(+), 61 deletions(-) diff --git a/boa_tester/src/exec/mod.rs b/boa_tester/src/exec/mod.rs index 38d285b64f..4d064732b0 100644 --- a/boa_tester/src/exec/mod.rs +++ b/boa_tester/src/exec/mod.rs @@ -3,8 +3,8 @@ mod js262; use crate::{ - read::ErrorType, Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, - TestResult, TestSuite, + read::ErrorType, Harness, Outcome, Phase, SpecVersion, Statistics, SuiteResult, Test, + TestFlags, TestOutcomeResult, TestResult, TestSuite, }; use boa_engine::{ context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction, @@ -57,53 +57,75 @@ impl TestSuite { println!(); } - // Count passed tests - let mut passed = 0; - let mut ignored = 0; - let mut panic = 0; + // Count passed tests and es specs + let mut all = Statistics::default(); + let mut es5 = Statistics::default(); + let mut es6 = Statistics::default(); + + let mut append_stats = |spec_version: SpecVersion, f: &dyn Fn(&mut Statistics)| { + f(&mut all); + if spec_version == SpecVersion::ES5 { + f(&mut es5); + } else if spec_version == SpecVersion::ES6 { + f(&mut es6); + } + }; + for test in &tests { match test.result { - TestOutcomeResult::Passed => passed += 1, - TestOutcomeResult::Ignored => ignored += 1, - TestOutcomeResult::Panic => panic += 1, + TestOutcomeResult::Passed => { + append_stats(test.spec_version, &|stats| { + stats.passed += 1; + }); + } + TestOutcomeResult::Ignored => { + append_stats(test.spec_version, &|stats| { + stats.ignored += 1; + }); + } + TestOutcomeResult::Panic => { + append_stats(test.spec_version, &|stats| { + stats.panic += 1; + }); + } TestOutcomeResult::Failed => {} } + append_stats(test.spec_version, &|stats| { + stats.total += 1; + }); } // Count total tests - let mut total = tests.len(); for suite in &suites { - total += suite.total; - passed += suite.passed; - ignored += suite.ignored; - panic += suite.panic; + all = all + suite.all_stats.clone(); + es5 = es5 + suite.es5_stats.clone(); + es6 = es6 + suite.es6_stats.clone(); features.append(&mut suite.features.clone()); } if verbose != 0 { println!( - "Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \ + "Suite {} results: total: {}, passed: {}, ignored: {}, failed: {} (panics: \ {}{}), conformance: {:.2}%", + all.total, self.path.display(), - passed.to_string().green(), - ignored.to_string().yellow(), - (total - passed - ignored).to_string().red(), - if panic == 0 { + all.passed.to_string().green(), + all.ignored.to_string().yellow(), + (all.total - all.passed - all.ignored).to_string().red(), + if all.panic == 0 { "0".normal() } else { - panic.to_string().red() + all.panic.to_string().red() }, - if panic == 0 { "" } else { " ⚠" }.red(), - (passed as f64 / total as f64) * 100.0 + if all.panic == 0 { "" } else { " ⚠" }.red(), + (all.passed as f64 / all.total as f64) * 100.0 ); } - SuiteResult { name: self.name.clone(), - total, - passed, - ignored, - panic, + all_stats: all, + es5_stats: es5, + es6_stats: es6, suites, tests, features, @@ -141,6 +163,7 @@ impl Test { } return TestResult { name: self.name.clone(), + spec_version: self.spec_version, strict, result: TestOutcomeResult::Failed, result_text: Box::from("Could not read test file.") @@ -159,6 +182,7 @@ impl Test { } return TestResult { name: self.name.clone(), + spec_version: self.spec_version, strict, result: TestOutcomeResult::Ignored, result_text: Box::default(), @@ -357,6 +381,7 @@ impl Test { TestResult { name: self.name.clone(), + spec_version: self.spec_version, strict, result, result_text: result_text.into_boxed_str(), diff --git a/boa_tester/src/main.rs b/boa_tester/src/main.rs index 4b9b30231b..1127648334 100644 --- a/boa_tester/src/main.rs +++ b/boa_tester/src/main.rs @@ -90,6 +90,7 @@ use serde::{ use std::{ fs::{self, File}, io::Read, + ops::Add, path::{Path, PathBuf}, }; @@ -270,21 +271,32 @@ fn run_test_suite( } let results = suite.run(&harness, verbose, parallel); + let total = results.all_stats.total; + let passed = results.all_stats.passed; + let ignored = results.all_stats.ignored; + let panicked = results.all_stats.panic; + println!(); println!("Results:"); - println!("Total tests: {}", results.total); - println!("Passed tests: {}", results.passed.to_string().green()); - println!("Ignored tests: {}", results.ignored.to_string().yellow()); + println!("Total tests: {total}"); + println!("Passed tests: {}", passed.to_string().green()); + println!("Ignored tests: {}", ignored.to_string().yellow()); println!( "Failed tests: {} (panics: {})", - (results.total - results.passed - results.ignored) - .to_string() - .red(), - results.panic.to_string().red() + (total - passed - ignored).to_string().red(), + panicked.to_string().red() ); println!( "Conformance: {:.2}%", - (results.passed as f64 / results.total as f64) * 100.0 + (passed as f64 / total as f64) * 100.0 + ); + println!( + "ES5 Conformance: {:.2}%", + (results.es5_stats.passed as f64 / results.es5_stats.total as f64) * 100.0 + ); + println!( + "ES6 Conformance: {:.2}%", + (results.es6_stats.passed as f64 / results.es6_stats.total as f64) * 100.0 ); write_json(results, output, verbose) @@ -318,12 +330,10 @@ struct TestSuite { tests: Box<[Test]>, } -/// Outcome of a test suite. -#[derive(Debug, Clone, Serialize, Deserialize)] -struct SuiteResult { - #[serde(rename = "n")] - name: Box, - #[serde(rename = "c")] +/// Represents a tests statistic +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +struct Statistics { + #[serde(rename = "t")] total: usize, #[serde(rename = "o")] passed: usize, @@ -331,6 +341,32 @@ struct SuiteResult { ignored: usize, #[serde(rename = "p")] panic: usize, +} + +impl Add for Statistics { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self { + total: self.total + rhs.total, + passed: self.passed + rhs.passed, + ignored: self.ignored + rhs.ignored, + panic: self.panic + rhs.panic, + } + } +} + +/// Outcome of a test suite. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SuiteResult { + #[serde(rename = "n")] + name: Box, + #[serde(rename = "a")] + all_stats: Statistics, + #[serde(rename = "a5", default)] + es5_stats: Statistics, + #[serde(rename = "a6", default)] + es6_stats: Statistics, #[serde(skip_serializing_if = "Vec::is_empty", default)] #[serde(rename = "s")] suites: Vec, @@ -348,6 +384,8 @@ struct SuiteResult { struct TestResult { #[serde(rename = "n")] name: Box, + #[serde(rename = "v", default)] + spec_version: SpecVersion, #[serde(rename = "s", default)] strict: bool, #[serde(skip)] @@ -368,6 +406,15 @@ enum TestOutcomeResult { Panic, } +#[derive(Debug, Serialize, Clone, Copy, Deserialize, PartialEq, Default)] +#[serde(untagged)] +enum SpecVersion { + ES5 = 5, + ES6 = 6, + #[default] + ES13 = 13, +} + /// Represents a test. #[derive(Debug, Clone)] #[allow(dead_code)] @@ -375,6 +422,7 @@ struct Test { name: Box, description: Box, esid: Option>, + spec_version: SpecVersion, flags: TestFlags, information: Box, features: Box<[Box]>, @@ -392,10 +440,19 @@ impl Test { N: Into>, C: Into>, { + let spec_version = if metadata.es5id.is_some() { + SpecVersion::ES5 + } else if metadata.es6id.is_some() { + SpecVersion::ES6 + } else { + SpecVersion::ES13 + }; + Self { name: name.into(), description: metadata.description, esid: metadata.esid, + spec_version, flags: metadata.flags.into(), information: metadata.info, features: metadata.features, diff --git a/boa_tester/src/results.rs b/boa_tester/src/results.rs index f1edfa7ebc..5fdcdac0ce 100644 --- a/boa_tester/src/results.rs +++ b/boa_tester/src/results.rs @@ -1,3 +1,5 @@ +use crate::Statistics; + use super::SuiteResult; use color_eyre::{eyre::WrapErr, Result}; use serde::{Deserialize, Serialize}; @@ -25,14 +27,12 @@ struct ReducedResultInfo { commit: Box, #[serde(rename = "u")] test262_commit: Box, - #[serde(rename = "t")] - total: usize, - #[serde(rename = "o")] - passed: usize, - #[serde(rename = "i")] - ignored: usize, - #[serde(rename = "p")] - panic: usize, + #[serde(rename = "a")] + all_stats: Statistics, + #[serde(rename = "a5", default)] + es5_stats: Statistics, + #[serde(rename = "a6", default)] + es6_stats: Statistics, } impl From for ReducedResultInfo { @@ -41,10 +41,9 @@ impl From for ReducedResultInfo { Self { commit: info.commit, test262_commit: info.test262_commit, - total: info.results.total, - passed: info.results.passed, - ignored: info.results.ignored, - panic: info.results.panic, + all_stats: info.results.all_stats, + es5_stats: info.results.es5_stats, + es6_stats: info.results.es6_stats, } } } @@ -220,24 +219,24 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result )) .wrap_err("could not read the new results")?; - let base_total = base_results.results.total as isize; - let new_total = new_results.results.total as isize; + let base_total = base_results.results.all_stats.total as isize; + let new_total = new_results.results.all_stats.total as isize; let total_diff = new_total - base_total; - let base_passed = base_results.results.passed as isize; - let new_passed = new_results.results.passed as isize; + let base_passed = base_results.results.all_stats.passed as isize; + let new_passed = new_results.results.all_stats.passed as isize; let passed_diff = new_passed - base_passed; - let base_ignored = base_results.results.ignored as isize; - let new_ignored = new_results.results.ignored as isize; + let base_ignored = base_results.results.all_stats.ignored as isize; + let new_ignored = new_results.results.all_stats.ignored as isize; let ignored_diff = new_ignored - base_ignored; let base_failed = base_total - base_passed - base_ignored; let new_failed = new_total - new_passed - new_ignored; let failed_diff = new_failed - base_failed; - let base_panics = base_results.results.panic as isize; - let new_panics = new_results.results.panic as isize; + let base_panics = base_results.results.all_stats.panic as isize; + let new_panics = new_results.results.all_stats.panic as isize; let panic_diff = new_panics - base_panics; let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;