Consider strict + no-strict tests as a single test (#3675)

test262_config.toml

@@ -1,4 +1,4 @@
-commit = "f742eb092df835fd07769bbb3537232d3efb61ed"
+commit = "6f7ae1f311a7b01ef2358de7f4f6fd42c3ae3839"
 
 [ignored]
 # Not implemented yet:

tests/tester/src/exec/mod.rs

@@ -73,13 +73,13 @@ impl TestSuite {
             self.tests
                 .par_iter()
                 .filter(|test| test.edition <= max_edition)
-                .flat_map(|test| test.run(harness, verbose, optimizer_options, console))
+                .map(|test| test.run(harness, verbose, optimizer_options, console))
                 .collect()
         } else {
             self.tests
                 .iter()
                 .filter(|test| test.edition <= max_edition)
-                .flat_map(|test| test.run(harness, verbose, optimizer_options, console))
+                .map(|test| test.run(harness, verbose, optimizer_options, console))
                 .collect()
         };
 
@@ -168,21 +168,29 @@ impl Test {
         verbose: u8,
         optimizer_options: OptimizerOptions,
         console: bool,
-    ) -> Vec<TestResult> {
+    ) -> TestResult {
-        let mut results = Vec::new();
+        if self.flags.contains(TestFlags::MODULE) || self.flags.contains(TestFlags::RAW) {
-        if self.flags.contains(TestFlags::MODULE) {
+            return self.run_once(harness, false, verbose, optimizer_options, console);
-            results.push(self.run_once(harness, false, verbose, optimizer_options, console));
-        } else {
-            if self.flags.contains(TestFlags::STRICT) && !self.flags.contains(TestFlags::RAW) {
-                results.push(self.run_once(harness, true, verbose, optimizer_options, console));
         }
 
-            if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) {
+        if self
-                results.push(self.run_once(harness, false, verbose, optimizer_options, console));
+            .flags
+            .contains(TestFlags::STRICT | TestFlags::NO_STRICT)
+        {
+            let r = self.run_once(harness, false, verbose, optimizer_options, console);
+            if r.result != TestOutcomeResult::Passed {
+                return r;
             }
+            self.run_once(harness, true, verbose, optimizer_options, console)
+        } else {
+            self.run_once(
+                harness,
+                self.flags.contains(TestFlags::STRICT),
+                verbose,
+                optimizer_options,
+                console,
+            )
         }
-
-        results
     }
 
     /// Runs the test once, in strict or non-strict mode
@@ -208,7 +216,6 @@ impl Test {
             return TestResult {
                 name: self.name.clone(),
                 edition: self.edition,
-                strict,
                 result: TestOutcomeResult::Failed,
                 result_text: Box::from("Could not read test file."),
             };
@@ -227,7 +234,6 @@ impl Test {
             return TestResult {
                 name: self.name.clone(),
                 edition: self.edition,
-                strict,
                 result: TestOutcomeResult::Ignored,
                 result_text: Box::default(),
             };
@@ -579,7 +585,6 @@ impl Test {
         TestResult {
             name: self.name.clone(),
             edition: self.edition,
-            strict,
             result,
             result_text: result_text.into_boxed_str(),
         }

tests/tester/src/main.rs

@@ -765,8 +765,6 @@ struct TestResult {
     name: Box<str>,
     #[serde(rename = "v", default)]
     edition: SpecEdition,
-    #[serde(rename = "s", default)]
-    strict: bool,
     #[serde(skip)]
     result_text: Box<str>,
     #[serde(rename = "r")]

tests/tester/src/results.rs

@@ -461,17 +461,12 @@ fn compute_result_diff(
         if let Some(new_test) = new_result
             .tests
             .iter()
-            .find(|new_test| new_test.strict == base_test.strict && new_test.name == base_test.name)
+            .find(|new_test| new_test.name == base_test.name)
         {
             let test_name = format!(
-                "test/{}/{}.js {}(previously {:?})",
+                "test/{}/{}.js (previously {:?})",
                 base.display(),
                 new_test.name,
-                if base_test.strict {
-                    "[strict mode] "
-                } else {
-                    ""
-                },
                 base_test.result
             )
             .into_boxed_str();