diff --git a/tests/tester/src/exec/mod.rs b/tests/tester/src/exec/mod.rs index f57d55aff89..8136425ba41 100644 --- a/tests/tester/src/exec/mod.rs +++ b/tests/tester/src/exec/mod.rs @@ -19,7 +19,7 @@ use boa_engine::{ }; use colored::Colorize; use rayon::prelude::*; -use rustc_hash::FxHashSet; +use rustc_hash::FxHashMap; use std::{cell::RefCell, eprintln, rc::Rc}; use self::js262::WorkerHandles; @@ -34,12 +34,12 @@ impl TestSuite { max_edition: SpecEdition, optimizer_options: OptimizerOptions, console: bool, - ) -> SuiteResult { + ) -> (Box, SuiteResult) { if verbose != 0 { println!("Suite {}:", self.path.display()); } - let suites: Vec<_> = if parallel { + let suites: FxHashMap<_, _> = if parallel { self.suites .par_iter() .map(|suite| { @@ -69,7 +69,7 @@ impl TestSuite { .collect() }; - let tests: Vec<_> = if parallel { + let tests: FxHashMap<_, _> = if parallel { self.tests .par_iter() .filter(|test| test.edition <= max_edition) @@ -83,11 +83,6 @@ impl TestSuite { .collect() }; - let mut features = FxHashSet::default(); - for test_iter in &*self.tests { - features.extend(test_iter.features.iter().map(ToString::to_string)); - } - if verbose != 0 { println!(); } @@ -96,27 +91,29 @@ impl TestSuite { let mut versioned_stats = VersionedStats::default(); let mut es_next = Statistics::default(); - for test in &tests { - match test.result { - TestOutcomeResult::Passed => { + for test in tests.values() { + match (test.strict, test.no_strict) { + (Some(TestOutcomeResult::Passed), None | Some(TestOutcomeResult::Passed)) + | (None, Some(TestOutcomeResult::Passed)) => { versioned_stats.apply(test.edition, |stats| { stats.passed += 1; }); es_next.passed += 1; } - TestOutcomeResult::Ignored => { + (Some(TestOutcomeResult::Ignored), _) | (_, Some(TestOutcomeResult::Ignored)) => { versioned_stats.apply(test.edition, |stats| { stats.ignored += 1; }); es_next.ignored += 1; } - TestOutcomeResult::Panic => { + (Some(TestOutcomeResult::Panic), _) | (_, Some(TestOutcomeResult::Panic)) => { versioned_stats.apply(test.edition, |stats| { stats.panic += 1; }); es_next.panic += 1; } - TestOutcomeResult::Failed => {} + (Some(TestOutcomeResult::Failed), _) | (_, Some(TestOutcomeResult::Failed)) => {} + (None, None) => unreachable!("test should have at least one result"), } versioned_stats.apply(test.edition, |stats| { stats.total += 1; @@ -125,10 +122,9 @@ impl TestSuite { } // Count total tests - for suite in &suites { + for suite in suites.values() { versioned_stats += suite.versioned_stats; es_next += suite.stats; - features.extend(suite.features.iter().cloned()); } if verbose != 0 { @@ -149,48 +145,51 @@ impl TestSuite { (es_next.passed as f64 / es_next.total as f64) * 100.0 ); } - SuiteResult { - name: self.name.clone(), - stats: es_next, - versioned_stats, - suites, - tests, - features, - } + ( + self.name.clone(), + SuiteResult { + stats: es_next, + versioned_stats, + suites, + tests, + }, + ) } } impl Test { /// Runs the test. + /// + /// Returns the test name and the result of the test. pub(crate) fn run( &self, harness: &Harness, verbose: u8, optimizer_options: OptimizerOptions, console: bool, - ) -> TestResult { + ) -> (Box, TestResult) { + let mut result = TestResult { + edition: self.edition, + strict: None, + no_strict: None, + }; + if self.flags.contains(TestFlags::MODULE) || self.flags.contains(TestFlags::RAW) { - return self.run_once(harness, false, verbose, optimizer_options, console); - } + result.no_strict = + Some(self.run_once(harness, false, verbose, optimizer_options, console)); + } else { + if self.flags.contains(TestFlags::STRICT) && !self.flags.contains(TestFlags::RAW) { + result.strict = + Some(self.run_once(harness, true, verbose, optimizer_options, console)); + } - if self - .flags - .contains(TestFlags::STRICT | TestFlags::NO_STRICT) - { - let r = self.run_once(harness, false, verbose, optimizer_options, console); - if r.result != TestOutcomeResult::Passed { - return r; + if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) { + result.no_strict = + Some(self.run_once(harness, false, verbose, optimizer_options, console)); } - self.run_once(harness, true, verbose, optimizer_options, console) - } else { - self.run_once( - harness, - self.flags.contains(TestFlags::STRICT), - verbose, - optimizer_options, - console, - ) } + + (self.name.clone(), result) } /// Runs the test once, in strict or non-strict mode @@ -201,7 +200,7 @@ impl Test { verbose: u8, optimizer_options: OptimizerOptions, console: bool, - ) -> TestResult { + ) -> TestOutcomeResult { let Ok(source) = Source::from_filepath(&self.path) else { if verbose > 1 { println!( @@ -213,12 +212,7 @@ impl Test { } else { print!("{}", "F".red()); } - return TestResult { - name: self.name.clone(), - edition: self.edition, - result: TestOutcomeResult::Failed, - result_text: Box::from("Could not read test file."), - }; + return TestOutcomeResult::Failed; }; if self.ignored { if verbose > 1 { @@ -231,12 +225,7 @@ impl Test { } else { print!("{}", "-".yellow()); } - return TestResult { - name: self.name.clone(), - edition: self.edition, - result: TestOutcomeResult::Ignored, - result_text: Box::default(), - }; + return TestOutcomeResult::Ignored; } if verbose > 1 { println!( @@ -582,12 +571,7 @@ impl Test { println!(); } - TestResult { - name: self.name.clone(), - edition: self.edition, - result, - result_text: result_text.into_boxed_str(), - } + result } /// Sets the environment up to run the test. diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index 8c703f176f8..0737d1235e0 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -451,7 +451,7 @@ fn run_test_suite( if verbose != 0 { println!("Test suite loaded, starting tests..."); } - let results = suite.run( + let (_name, results) = suite.run( &harness, verbose, parallel, @@ -590,15 +590,25 @@ impl AddAssign for Statistics { /// Represents tests statistics separated by ECMAScript edition #[derive(Default, Debug, Copy, Clone, Serialize)] struct VersionedStats { + #[serde(rename = "5")] es5: Statistics, + #[serde(rename = "6")] es6: Statistics, + #[serde(rename = "7")] es7: Statistics, + #[serde(rename = "8")] es8: Statistics, + #[serde(rename = "9")] es9: Statistics, + #[serde(rename = "10")] es10: Statistics, + #[serde(rename = "11")] es11: Statistics, + #[serde(rename = "12")] es12: Statistics, + #[serde(rename = "13")] es13: Statistics, + #[serde(rename = "14")] es14: Statistics, } @@ -609,16 +619,26 @@ impl<'de> Deserialize<'de> for VersionedStats { { #[derive(Deserialize)] struct Inner { + #[serde(rename = "5")] es5: Statistics, + #[serde(rename = "6")] es6: Statistics, + #[serde(rename = "7")] es7: Statistics, + #[serde(rename = "8")] es8: Statistics, + #[serde(rename = "9")] es9: Statistics, + #[serde(rename = "10")] es10: Statistics, + #[serde(rename = "11")] es11: Statistics, + #[serde(rename = "12")] es12: Statistics, + #[serde(rename = "13")] es13: Statistics, #[serde(default)] + #[serde(rename = "14")] es14: Option, } @@ -740,35 +760,28 @@ impl AddAssign for VersionedStats { /// Outcome of a test suite. #[derive(Debug, Clone, Serialize, Deserialize)] struct SuiteResult { - #[serde(rename = "n")] - name: Box, #[serde(rename = "a")] stats: Statistics, - #[serde(rename = "av", default)] + #[serde(rename = "v", default)] versioned_stats: VersionedStats, - #[serde(skip_serializing_if = "Vec::is_empty", default)] + #[serde(skip_serializing_if = "FxHashMap::is_empty", default)] #[serde(rename = "s")] - suites: Vec, - #[serde(skip_serializing_if = "Vec::is_empty", default)] + suites: FxHashMap, SuiteResult>, + #[serde(skip_serializing_if = "FxHashMap::is_empty", default)] #[serde(rename = "t")] - tests: Vec, - #[serde(skip_serializing_if = "FxHashSet::is_empty", default)] - #[serde(rename = "f")] - features: FxHashSet, + tests: FxHashMap, TestResult>, } -/// Outcome of a test. +/// Result of a test, including the outcome for strict and non-strict mode. #[derive(Debug, Clone, Serialize, Deserialize)] #[allow(dead_code)] struct TestResult { - #[serde(rename = "n")] - name: Box, #[serde(rename = "v", default)] edition: SpecEdition, - #[serde(skip)] - result_text: Box, - #[serde(rename = "r")] - result: TestOutcomeResult, + #[serde(rename = "s", default)] + strict: Option, + #[serde(rename = "r", default)] + no_strict: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/tests/tester/src/results.rs b/tests/tester/src/results.rs index 12600d1ca17..a1e7077d850 100644 --- a/tests/tester/src/results.rs +++ b/tests/tester/src/results.rs @@ -1,8 +1,8 @@ -use crate::{Statistics, VersionedStats}; +use crate::{Statistics, TestOutcomeResult, VersionedStats}; use super::SuiteResult; use color_eyre::{eyre::WrapErr, Result}; -use rustc_hash::FxHashSet; +use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use std::{ env, fs, @@ -17,8 +17,12 @@ struct ResultInfo { commit: Box, #[serde(rename = "u")] test262_commit: Box, + #[serde(rename = "a")] + stats: Statistics, + #[serde(rename = "v")] + versioned_results: VersionedStats, #[serde(rename = "r")] - results: SuiteResult, + results: FxHashMap, SuiteResult>, } /// Structure to store full result information. @@ -30,8 +34,6 @@ struct ReducedResultInfo { test262_commit: Box, #[serde(rename = "a")] stats: Statistics, - #[serde(rename = "av", default)] - versioned_stats: VersionedStats, } impl From for ReducedResultInfo { @@ -40,31 +42,7 @@ impl From for ReducedResultInfo { Self { commit: info.commit, test262_commit: info.test262_commit, - stats: info.results.stats, - versioned_stats: info.results.versioned_stats, - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -struct FeaturesInfo { - #[serde(rename = "c")] - commit: Box, - #[serde(rename = "u")] - test262_commit: Box, - #[serde(rename = "n")] - suite_name: Box, - #[serde(rename = "f")] - features: FxHashSet, -} - -impl From for FeaturesInfo { - fn from(info: ResultInfo) -> Self { - Self { - commit: info.commit, - test262_commit: info.test262_commit, - suite_name: info.results.name, - features: info.results.features, + stats: info.stats, } } } @@ -75,9 +53,6 @@ const LATEST_FILE_NAME: &str = "latest.json"; /// File name of the "all results" JSON file. const RESULTS_FILE_NAME: &str = "results.json"; -/// File name of the "features" JSON file. -const FEATURES_FILE_NAME: &str = "features.json"; - /// Writes the results of running the test suite to the given JSON output file. /// /// It will append the results to the ones already present, in an array. @@ -95,13 +70,13 @@ pub(crate) fn write_json( let output_dir = if branch.is_empty() { output_dir.to_path_buf() } else { - let folder = output_dir.join(branch); + let folder = output_dir.join(&branch); fs::create_dir_all(&folder)?; folder }; // We make sure we are using the latest commit information in GitHub pages: - update_gh_pages_repo(output_dir.as_path(), verbose); + update_gh_pages_repo(output_dir.as_path(), &branch, verbose); if verbose != 0 { println!("Writing the results to {}...", output_dir.display()); @@ -113,49 +88,35 @@ pub(crate) fn write_json( let new_results = ResultInfo { commit: env::var("GITHUB_SHA").unwrap_or_default().into_boxed_str(), - test262_commit: get_test262_commit(test262_path)?, - results, + test262_commit: get_test262_commit(test262_path) + .context("could not get the test262 commit")?, + stats: results.stats, + versioned_results: results.versioned_stats, + results: results.suites, }; let latest = BufWriter::new(fs::File::create(latest)?); serde_json::to_writer(latest, &new_results)?; - // Write the full list of results, retrieving the existing ones first. - - let all_path = output_dir.join(RESULTS_FILE_NAME); - - let mut all_results: Vec = if all_path.exists() { - serde_json::from_reader(BufReader::new(fs::File::open(&all_path)?))? - } else { - Vec::new() - }; + // Write the full result history for "main" + if branch == "main" { + let all_path = output_dir.join(RESULTS_FILE_NAME); - all_results.push(new_results.clone().into()); + // We only keep history for the main branch + let mut all_results: Vec = if all_path.is_file() { + serde_json::from_reader(BufReader::new(fs::File::open(&all_path)?))? + } else { + Vec::new() + }; - let output = BufWriter::new(fs::File::create(&all_path)?); - serde_json::to_writer(output, &all_results)?; + all_results.push(new_results.into()); - if verbose != 0 { - println!("Results written correctly"); + let output = BufWriter::new(fs::File::create(&all_path)?); + serde_json::to_writer(output, &all_results)?; } - // Write the full list of features, existing features go first. - - let features = output_dir.join(FEATURES_FILE_NAME); - - let mut all_features: Vec = if features.exists() { - serde_json::from_reader(BufReader::new(fs::File::open(&features)?))? - } else { - Vec::new() - }; - - all_features.push(new_results.into()); - - let features = BufWriter::new(fs::File::create(&features)?); - serde_json::to_writer(features, &all_features)?; - if verbose != 0 { - println!("Features written correctly"); + println!("Results written correctly"); } Ok(()) @@ -171,17 +132,20 @@ fn get_test262_commit(test262_path: &Path) -> Result> { } /// Updates the GitHub pages repository by pulling latest changes before writing the new things. -fn update_gh_pages_repo(path: &Path, verbose: u8) { - if env::var("GITHUB_REF").is_ok() { - use std::process::Command; +fn update_gh_pages_repo(path: &Path, branch: &str, verbose: u8) { + use std::process::Command; - // We run the command to pull the gh-pages branch: git -C ../gh-pages/ pull origin - Command::new("git") - .args(["-C", "../gh-pages", "pull", "--ff-only"]) - .output() - .expect("could not update GitHub Pages"); + // We run the command to pull the gh-pages branch: git -C ../gh-pages/ pull --ff-only + if verbose != 0 { + println!("Cloning the Github Pages branch in ../gh-pages/..."); + } + Command::new("git") + .args(["-C", "../gh-pages", "pull", "--ff-only"]) + .output() + .expect("could not update GitHub Pages"); - // Copy the full results file + if branch == "main" { + // Copy the full results file if in the main branch let from = Path::new("../gh-pages/test262/refs/heads/main/").join(RESULTS_FILE_NAME); let to = path.join(RESULTS_FILE_NAME); @@ -210,31 +174,41 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result )) .wrap_err("could not read the new results")?; - let base_total = base_results.results.stats.total as isize; - let new_total = new_results.results.stats.total as isize; + let base_total = base_results.stats.total as isize; + let new_total = new_results.stats.total as isize; let total_diff = new_total - base_total; - let base_passed = base_results.results.stats.passed as isize; - let new_passed = new_results.results.stats.passed as isize; + let base_passed = base_results.stats.passed as isize; + let new_passed = new_results.stats.passed as isize; let passed_diff = new_passed - base_passed; - let base_ignored = base_results.results.stats.ignored as isize; - let new_ignored = new_results.results.stats.ignored as isize; + let base_ignored = base_results.stats.ignored as isize; + let new_ignored = new_results.stats.ignored as isize; let ignored_diff = new_ignored - base_ignored; let base_failed = base_total - base_passed - base_ignored; let new_failed = new_total - new_passed - new_ignored; let failed_diff = new_failed - base_failed; - let base_panics = base_results.results.stats.panic as isize; - let new_panics = new_results.results.stats.panic as isize; + let base_panics = base_results.stats.panic as isize; + let new_panics = new_results.stats.panic as isize; let panic_diff = new_panics - base_panics; let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64; let new_conformance = (new_passed as f64 / new_total as f64) * 100_f64; let conformance_diff = new_conformance - base_conformance; - let test_diff = compute_result_diff(Path::new(""), &base_results.results, &new_results.results); + let test_diff = std::iter::zip(base_results.results, new_results.results).fold( + ResultDiff::default(), + |mut diff, ((base_name, base_suite), (_new_name, new_suite))| { + diff.extend(compute_result_diff( + Path::new(base_name.as_ref()), + &base_suite, + &new_suite, + )); + diff + }, + ); if markdown { /// Simple function to add commas as thousands separator for integers. @@ -453,45 +427,37 @@ fn compute_result_diff( base_result: &SuiteResult, new_result: &SuiteResult, ) -> ResultDiff { - use super::TestOutcomeResult; - let mut final_diff = ResultDiff::default(); - for base_test in &base_result.tests { - if let Some(new_test) = new_result + for (base_name, base_test) in &base_result.tests { + if let Some((new_name, new_test)) = new_result .tests .iter() - .find(|new_test| new_test.name == base_test.name) + .find(|(new_name, _new_test)| new_name == &base_name) { - let test_name = format!( - "test/{}/{}.js (previously {:?})", - base.display(), - new_test.name, - base_test.result - ) - .into_boxed_str(); + let test_name = format!("test/{}/{}.js", base.display(), new_name); - match (base_test.result, new_test.result) { - (a, b) if a == b => {} - (TestOutcomeResult::Ignored, TestOutcomeResult::Failed) => {} + if let (Some(base_result), Some(new_result)) = (base_test.strict, new_test.strict) { + let test_name = format!("{test_name} [strict mode] (previously {base_result:?})"); - (_, TestOutcomeResult::Passed) => final_diff.fixed.push(test_name), - (TestOutcomeResult::Panic, _) => final_diff.panic_fixes.push(test_name), - (_, TestOutcomeResult::Failed) => final_diff.broken.push(test_name), - (_, TestOutcomeResult::Panic) => final_diff.new_panics.push(test_name), + add_results(&mut final_diff, test_name, base_result, new_result); + } + + if let (Some(base_result), Some(new_result)) = (base_test.strict, new_test.strict) { + let test_name = format!("{test_name} (previously {base_result:?})"); - _ => {} + add_results(&mut final_diff, test_name, base_result, new_result); } } } - for base_suite in &base_result.suites { - if let Some(new_suite) = new_result + for (base_name, base_suite) in &base_result.suites { + if let Some((new_name, new_suite)) = new_result .suites .iter() - .find(|new_suite| new_suite.name == base_suite.name) + .find(|(new_name, _new_suite)| new_name == &base_name) { - let new_base = base.join(new_suite.name.as_ref()); + let new_base = base.join(new_name.as_ref()); let diff = compute_result_diff(new_base.as_path(), base_suite, new_suite); final_diff.extend(diff); @@ -500,3 +466,27 @@ fn compute_result_diff( final_diff } + +/// Adds the results to the final diff. +fn add_results( + final_diff: &mut ResultDiff, + test_name: N, + base_result: TestOutcomeResult, + new_result: TestOutcomeResult, +) where + N: Into>, +{ + let test_name = test_name.into(); + + match (base_result, new_result) { + (a, b) if a == b => {} + (TestOutcomeResult::Ignored, TestOutcomeResult::Failed) => {} + + (_, TestOutcomeResult::Passed) => final_diff.fixed.push(test_name), + (TestOutcomeResult::Panic, _) => final_diff.panic_fixes.push(test_name), + (_, TestOutcomeResult::Failed) => final_diff.broken.push(test_name), + (_, TestOutcomeResult::Panic) => final_diff.new_panics.push(test_name), + + _ => {} + } +}