github
/
boa
mirror of https://github.com/boa-dev/boa.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Add ES5 and ES6 Conformance calculation to boa_tester (#2690) 

This is calculated based the tests `es5id` or `es6id`.  

Judging by [this](https://github.com/tc39/test262/issues/1557) it's probably not the best method of doing it, but the alternative would require a lot of rework on the boa_tester so that it can pull an older version of the test262 spec which has it's own problems since there's not really an "ES5" only version.
 
I would think that if we're 100% passing on es5id's then it's safe to assume boa supports es5 (assuming test262's es5id is covering all the es5 cases)

This Pull Request fixes/closes #2629.

It changes the following:
- Store `spec_version` in TestResult based on the tests `es[6/5]id`
- Count all es5, es6 and their test outcome during `TestSuite::run()`
- Print the conformance.

I'm serializing the `spec_version` outcomes so that it can be displayed in test262 github page. I'd like to work on that too if possible. Let me know if there's anything more I should cover in this PR, I'll gladly do it :)

Co-authored-by: jedel1043 <jedel0124@gmail.com>
pull/2712/head
Zack Mitkin 2 years ago
parent
431a358b2f
commit
02f737aac7
3 changed files with 142 additions and 61 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 79
      boa_tester/src/exec/mod.rs
  2. 85
      boa_tester/src/main.rs
  3. 39
      boa_tester/src/results.rs

79
boa_tester/src/exec/mod.rs
Unescape View File

@ -3,8 +3,8 @@
mod js262; mod js262;
use crate::{ use crate::{
read::ErrorType, Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, read::ErrorType, Harness, Outcome, Phase, SpecVersion, Statistics, SuiteResult, Test,
TestResult, TestSuite, TestFlags, TestOutcomeResult, TestResult, TestSuite,
}; };
use boa_engine::{ use boa_engine::{
context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction, context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction,
@ -57,53 +57,75 @@ impl TestSuite {
println!(); println!();
} }
// Count passed tests // Count passed tests and es specs
let mut passed = 0; let mut all = Statistics::default();
let mut ignored = 0; let mut es5 = Statistics::default();
let mut panic = 0; let mut es6 = Statistics::default();
let mut append_stats = |spec_version: SpecVersion, f: &dyn Fn(&mut Statistics)| {
f(&mut all);
if spec_version == SpecVersion::ES5 {
f(&mut es5);
} else if spec_version == SpecVersion::ES6 {
f(&mut es6);
}
};
for test in &tests { for test in &tests {
match test.result { match test.result {
TestOutcomeResult::Passed => passed += 1, TestOutcomeResult::Passed => {
TestOutcomeResult::Ignored => ignored += 1, append_stats(test.spec_version, &|stats| {
TestOutcomeResult::Panic => panic += 1, stats.passed += 1;
});
}
TestOutcomeResult::Ignored => {
append_stats(test.spec_version, &|stats| {
stats.ignored += 1;
});
}
TestOutcomeResult::Panic => {
append_stats(test.spec_version, &|stats| {
stats.panic += 1;
});
}
TestOutcomeResult::Failed => {} TestOutcomeResult::Failed => {}
} }
append_stats(test.spec_version, &|stats| {
stats.total += 1;
});
} }
// Count total tests // Count total tests
let mut total = tests.len();
for suite in &suites { for suite in &suites {
total += suite.total; all = all + suite.all_stats.clone();
passed += suite.passed; es5 = es5 + suite.es5_stats.clone();
ignored += suite.ignored; es6 = es6 + suite.es6_stats.clone();
panic += suite.panic;
features.append(&mut suite.features.clone()); features.append(&mut suite.features.clone());
} }
if verbose != 0 { if verbose != 0 {
println!( println!(
"Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \ "Suite {} results: total: {}, passed: {}, ignored: {}, failed: {} (panics: \
{}{}), conformance: {:.2}%", {}{}), conformance: {:.2}%",
all.total,
self.path.display(), self.path.display(),
passed.to_string().green(), all.passed.to_string().green(),
ignored.to_string().yellow(), all.ignored.to_string().yellow(),
(total - passed - ignored).to_string().red(), (all.total - all.passed - all.ignored).to_string().red(),
if panic == 0 { if all.panic == 0 {
"0".normal() "0".normal()
} else { } else {
panic.to_string().red() all.panic.to_string().red()
}, },
if panic == 0 { "" } else { " ⚠" }.red(), if all.panic == 0 { "" } else { " ⚠" }.red(),
(passed as f64 / total as f64) * 100.0 (all.passed as f64 / all.total as f64) * 100.0
); );
} }
SuiteResult { SuiteResult {
name: self.name.clone(), name: self.name.clone(),
total, all_stats: all,
passed, es5_stats: es5,
ignored, es6_stats: es6,
panic,
suites, suites,
tests, tests,
features, features,
@ -141,6 +163,7 @@ impl Test {
} }
return TestResult { return TestResult {
name: self.name.clone(), name: self.name.clone(),
spec_version: self.spec_version,
strict, strict,
result: TestOutcomeResult::Failed, result: TestOutcomeResult::Failed,
result_text: Box::from("Could not read test file.") result_text: Box::from("Could not read test file.")
@ -159,6 +182,7 @@ impl Test {
} }
return TestResult { return TestResult {
name: self.name.clone(), name: self.name.clone(),
spec_version: self.spec_version,
strict, strict,
result: TestOutcomeResult::Ignored, result: TestOutcomeResult::Ignored,
result_text: Box::default(), result_text: Box::default(),
@ -357,6 +381,7 @@ impl Test {
TestResult { TestResult {
name: self.name.clone(), name: self.name.clone(),
spec_version: self.spec_version,
strict, strict,
result, result,
result_text: result_text.into_boxed_str(), result_text: result_text.into_boxed_str(),

85
boa_tester/src/main.rs
Unescape View File

@ -90,6 +90,7 @@ use serde::{
use std::{ use std::{
fs::{self, File}, fs::{self, File},
io::Read, io::Read,
ops::Add,
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
@ -270,21 +271,32 @@ fn run_test_suite(
} }
let results = suite.run(&harness, verbose, parallel); let results = suite.run(&harness, verbose, parallel);
let total = results.all_stats.total;
let passed = results.all_stats.passed;
let ignored = results.all_stats.ignored;
let panicked = results.all_stats.panic;
println!(); println!();
println!("Results:"); println!("Results:");
println!("Total tests: {}", results.total); println!("Total tests: {total}");
println!("Passed tests: {}", results.passed.to_string().green()); println!("Passed tests: {}", passed.to_string().green());
println!("Ignored tests: {}", results.ignored.to_string().yellow()); println!("Ignored tests: {}", ignored.to_string().yellow());
println!( println!(
"Failed tests: {} (panics: {})", "Failed tests: {} (panics: {})",
(results.total - results.passed - results.ignored) (total - passed - ignored).to_string().red(),
.to_string() panicked.to_string().red()
.red(),
results.panic.to_string().red()
); );
println!( println!(
"Conformance: {:.2}%", "Conformance: {:.2}%",
(results.passed as f64 / results.total as f64) * 100.0 (passed as f64 / total as f64) * 100.0
);
println!(
"ES5 Conformance: {:.2}%",
(results.es5_stats.passed as f64 / results.es5_stats.total as f64) * 100.0
);
println!(
"ES6 Conformance: {:.2}%",
(results.es6_stats.passed as f64 / results.es6_stats.total as f64) * 100.0
); );
write_json(results, output, verbose) write_json(results, output, verbose)
@ -318,12 +330,10 @@ struct TestSuite {
tests: Box<[Test]>, tests: Box<[Test]>,
} }
/// Outcome of a test suite. /// Represents a tests statistic
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Default, Debug, Clone, Serialize, Deserialize)]
struct SuiteResult { struct Statistics {
#[serde(rename = "n")] #[serde(rename = "t")]
name: Box<str>,
#[serde(rename = "c")]
total: usize, total: usize,
#[serde(rename = "o")] #[serde(rename = "o")]
passed: usize, passed: usize,
@ -331,6 +341,32 @@ struct SuiteResult {
ignored: usize, ignored: usize,
#[serde(rename = "p")] #[serde(rename = "p")]
panic: usize, panic: usize,
}
impl Add for Statistics {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self {
total: self.total + rhs.total,
passed: self.passed + rhs.passed,
ignored: self.ignored + rhs.ignored,
panic: self.panic + rhs.panic,
}
}
}
/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "a")]
all_stats: Statistics,
#[serde(rename = "a5", default)]
es5_stats: Statistics,
#[serde(rename = "a6", default)]
es6_stats: Statistics,
#[serde(skip_serializing_if = "Vec::is_empty", default)] #[serde(skip_serializing_if = "Vec::is_empty", default)]
#[serde(rename = "s")] #[serde(rename = "s")]
suites: Vec<SuiteResult>, suites: Vec<SuiteResult>,
@ -348,6 +384,8 @@ struct SuiteResult {
struct TestResult { struct TestResult {
#[serde(rename = "n")] #[serde(rename = "n")]
name: Box<str>, name: Box<str>,
#[serde(rename = "v", default)]
spec_version: SpecVersion,
#[serde(rename = "s", default)] #[serde(rename = "s", default)]
strict: bool, strict: bool,
#[serde(skip)] #[serde(skip)]
@ -368,6 +406,15 @@ enum TestOutcomeResult {
Panic, Panic,
} }
#[derive(Debug, Serialize, Clone, Copy, Deserialize, PartialEq, Default)]
#[serde(untagged)]
enum SpecVersion {
ES5 = 5,
ES6 = 6,
#[default]
ES13 = 13,
}
/// Represents a test. /// Represents a test.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
#[allow(dead_code)] #[allow(dead_code)]
@ -375,6 +422,7 @@ struct Test {
name: Box<str>, name: Box<str>,
description: Box<str>, description: Box<str>,
esid: Option<Box<str>>, esid: Option<Box<str>>,
spec_version: SpecVersion,
flags: TestFlags, flags: TestFlags,
information: Box<str>, information: Box<str>,
features: Box<[Box<str>]>, features: Box<[Box<str>]>,
@ -392,10 +440,19 @@ impl Test {
N: Into<Box<str>>, N: Into<Box<str>>,
C: Into<Box<Path>>, C: Into<Box<Path>>,
{ {
let spec_version = if metadata.es5id.is_some() {
SpecVersion::ES5
} else if metadata.es6id.is_some() {
SpecVersion::ES6
} else {
SpecVersion::ES13
};
Self { Self {
name: name.into(), name: name.into(),
description: metadata.description, description: metadata.description,
esid: metadata.esid, esid: metadata.esid,
spec_version,
flags: metadata.flags.into(), flags: metadata.flags.into(),
information: metadata.info, information: metadata.info,
features: metadata.features, features: metadata.features,

39
boa_tester/src/results.rs
Unescape View File

@ -1,3 +1,5 @@
use crate::Statistics;
use super::SuiteResult; use super::SuiteResult;
use color_eyre::{eyre::WrapErr, Result}; use color_eyre::{eyre::WrapErr, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -25,14 +27,12 @@ struct ReducedResultInfo {
commit: Box<str>, commit: Box<str>,
#[serde(rename = "u")] #[serde(rename = "u")]
test262_commit: Box<str>, test262_commit: Box<str>,
#[serde(rename = "t")] #[serde(rename = "a")]
total: usize, all_stats: Statistics,
#[serde(rename = "o")] #[serde(rename = "a5", default)]
passed: usize, es5_stats: Statistics,
#[serde(rename = "i")] #[serde(rename = "a6", default)]
ignored: usize, es6_stats: Statistics,
#[serde(rename = "p")]
panic: usize,
} }
impl From<ResultInfo> for ReducedResultInfo { impl From<ResultInfo> for ReducedResultInfo {
@ -41,10 +41,9 @@ impl From<ResultInfo> for ReducedResultInfo {
Self { Self {
commit: info.commit, commit: info.commit,
test262_commit: info.test262_commit, test262_commit: info.test262_commit,
total: info.results.total, all_stats: info.results.all_stats,
passed: info.results.passed, es5_stats: info.results.es5_stats,
ignored: info.results.ignored, es6_stats: info.results.es6_stats,
panic: info.results.panic,
} }
} }
} }
@ -220,24 +219,24 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result
)) ))
.wrap_err("could not read the new results")?; .wrap_err("could not read the new results")?;
let base_total = base_results.results.total as isize; let base_total = base_results.results.all_stats.total as isize;
let new_total = new_results.results.total as isize; let new_total = new_results.results.all_stats.total as isize;
let total_diff = new_total - base_total; let total_diff = new_total - base_total;
let base_passed = base_results.results.passed as isize; let base_passed = base_results.results.all_stats.passed as isize;
let new_passed = new_results.results.passed as isize; let new_passed = new_results.results.all_stats.passed as isize;
let passed_diff = new_passed - base_passed; let passed_diff = new_passed - base_passed;
let base_ignored = base_results.results.ignored as isize; let base_ignored = base_results.results.all_stats.ignored as isize;
let new_ignored = new_results.results.ignored as isize; let new_ignored = new_results.results.all_stats.ignored as isize;
let ignored_diff = new_ignored - base_ignored; let ignored_diff = new_ignored - base_ignored;
let base_failed = base_total - base_passed - base_ignored; let base_failed = base_total - base_passed - base_ignored;
let new_failed = new_total - new_passed - new_ignored; let new_failed = new_total - new_passed - new_ignored;
let failed_diff = new_failed - base_failed; let failed_diff = new_failed - base_failed;
let base_panics = base_results.results.panic as isize; let base_panics = base_results.results.all_stats.panic as isize;
let new_panics = new_results.results.panic as isize; let new_panics = new_results.results.all_stats.panic as isize;
let panic_diff = new_panics - base_panics; let panic_diff = new_panics - base_panics;
let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64; let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;

Write Preview
Loading…
Cancel
Save