Skip to content

Commit

Permalink
Fixed conformance reporting
Browse files Browse the repository at this point in the history
  • Loading branch information
@Razican
Razican committed Feb 24, 2024
1 parent c94e10d commit 3a8360b
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 190 deletions.
110 changes: 47 additions & 63 deletions tests/tester/src/exec/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use boa_engine::{
};
use colored::Colorize;
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use rustc_hash::FxHashMap;
use std::{cell::RefCell, eprintln, rc::Rc};

use self::js262::WorkerHandles;
Expand All @@ -34,12 +34,12 @@ impl TestSuite {
max_edition: SpecEdition,
optimizer_options: OptimizerOptions,
console: bool,
) -> SuiteResult {
) -> (Box<str>, SuiteResult) {
if verbose != 0 {
println!("Suite {}:", self.path.display());
}

let suites: Vec<_> = if parallel {
let suites: FxHashMap<_, _> = if parallel {
self.suites
.par_iter()
.map(|suite| {
Expand Down Expand Up @@ -69,7 +69,7 @@ impl TestSuite {
.collect()
};

let tests: Vec<_> = if parallel {
let tests: FxHashMap<_, _> = if parallel {
self.tests
.par_iter()
.filter(|test| test.edition <= max_edition)
Expand All @@ -83,11 +83,6 @@ impl TestSuite {
.collect()
};

let mut features = FxHashSet::default();
for test_iter in &*self.tests {
features.extend(test_iter.features.iter().map(ToString::to_string));
}

if verbose != 0 {
println!();
}
Expand All @@ -96,27 +91,29 @@ impl TestSuite {
let mut versioned_stats = VersionedStats::default();
let mut es_next = Statistics::default();

for test in &tests {
match test.result {
TestOutcomeResult::Passed => {
for test in tests.values() {
match (test.strict, test.no_strict) {
(Some(TestOutcomeResult::Passed), None | Some(TestOutcomeResult::Passed))
| (None, Some(TestOutcomeResult::Passed)) => {
versioned_stats.apply(test.edition, |stats| {
stats.passed += 1;
});
es_next.passed += 1;
}
TestOutcomeResult::Ignored => {
(Some(TestOutcomeResult::Ignored), _) | (_, Some(TestOutcomeResult::Ignored)) => {
versioned_stats.apply(test.edition, |stats| {
stats.ignored += 1;
});
es_next.ignored += 1;
}
TestOutcomeResult::Panic => {
(Some(TestOutcomeResult::Panic), _) | (_, Some(TestOutcomeResult::Panic)) => {
versioned_stats.apply(test.edition, |stats| {
stats.panic += 1;
});
es_next.panic += 1;
}
TestOutcomeResult::Failed => {}
(Some(TestOutcomeResult::Failed), _) | (_, Some(TestOutcomeResult::Failed)) => {}
(None, None) => unreachable!("test should have at least one result"),
}
versioned_stats.apply(test.edition, |stats| {
stats.total += 1;
Expand All @@ -125,10 +122,9 @@ impl TestSuite {
}

// Count total tests
for suite in &suites {
for suite in suites.values() {
versioned_stats += suite.versioned_stats;
es_next += suite.stats;
features.extend(suite.features.iter().cloned());
}

if verbose != 0 {
Expand All @@ -149,48 +145,51 @@ impl TestSuite {
(es_next.passed as f64 / es_next.total as f64) * 100.0
);
}
SuiteResult {
name: self.name.clone(),
stats: es_next,
versioned_stats,
suites,
tests,
features,
}
(
self.name.clone(),
SuiteResult {
stats: es_next,
versioned_stats,
suites,
tests,
},
)
}
}

impl Test {
/// Runs the test.
///
/// Returns the test name and the result of the test.
pub(crate) fn run(
&self,
harness: &Harness,
verbose: u8,
optimizer_options: OptimizerOptions,
console: bool,
) -> TestResult {
) -> (Box<str>, TestResult) {
let mut result = TestResult {
edition: self.edition,
strict: None,
no_strict: None,
};

if self.flags.contains(TestFlags::MODULE) || self.flags.contains(TestFlags::RAW) {
return self.run_once(harness, false, verbose, optimizer_options, console);
}
result.no_strict =
Some(self.run_once(harness, false, verbose, optimizer_options, console));
} else {
if self.flags.contains(TestFlags::STRICT) && !self.flags.contains(TestFlags::RAW) {
result.strict =
Some(self.run_once(harness, true, verbose, optimizer_options, console));
}

if self
.flags
.contains(TestFlags::STRICT | TestFlags::NO_STRICT)
{
let r = self.run_once(harness, false, verbose, optimizer_options, console);
if r.result != TestOutcomeResult::Passed {
return r;
if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) {
result.no_strict =
Some(self.run_once(harness, false, verbose, optimizer_options, console));
}
self.run_once(harness, true, verbose, optimizer_options, console)
} else {
self.run_once(
harness,
self.flags.contains(TestFlags::STRICT),
verbose,
optimizer_options,
console,
)
}

(self.name.clone(), result)
}

/// Runs the test once, in strict or non-strict mode
Expand All @@ -201,7 +200,7 @@ impl Test {
verbose: u8,
optimizer_options: OptimizerOptions,
console: bool,
) -> TestResult {
) -> TestOutcomeResult {
let Ok(source) = Source::from_filepath(&self.path) else {
if verbose > 1 {
println!(
Expand All @@ -213,12 +212,7 @@ impl Test {
} else {
print!("{}", "F".red());
}
return TestResult {
name: self.name.clone(),
edition: self.edition,
result: TestOutcomeResult::Failed,
result_text: Box::from("Could not read test file."),
};
return TestOutcomeResult::Failed;
};
if self.ignored {
if verbose > 1 {
Expand All @@ -231,12 +225,7 @@ impl Test {
} else {
print!("{}", "-".yellow());
}
return TestResult {
name: self.name.clone(),
edition: self.edition,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
};
return TestOutcomeResult::Ignored;
}
if verbose > 1 {
println!(
Expand Down Expand Up @@ -582,12 +571,7 @@ impl Test {
println!();
}

TestResult {
name: self.name.clone(),
edition: self.edition,
result,
result_text: result_text.into_boxed_str(),
}
result
}

/// Sets the environment up to run the test.
Expand Down
49 changes: 31 additions & 18 deletions tests/tester/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ fn run_test_suite(
if verbose != 0 {
println!("Test suite loaded, starting tests...");
}
let results = suite.run(
let (_name, results) = suite.run(
&harness,
verbose,
parallel,
Expand Down Expand Up @@ -590,15 +590,25 @@ impl AddAssign for Statistics {
/// Represents tests statistics separated by ECMAScript edition
#[derive(Default, Debug, Copy, Clone, Serialize)]
struct VersionedStats {
#[serde(rename = "5")]
es5: Statistics,
#[serde(rename = "6")]
es6: Statistics,
#[serde(rename = "7")]
es7: Statistics,
#[serde(rename = "8")]
es8: Statistics,
#[serde(rename = "9")]
es9: Statistics,
#[serde(rename = "10")]
es10: Statistics,
#[serde(rename = "11")]
es11: Statistics,
#[serde(rename = "12")]
es12: Statistics,
#[serde(rename = "13")]
es13: Statistics,
#[serde(rename = "14")]
es14: Statistics,
}

Expand All @@ -609,16 +619,26 @@ impl<'de> Deserialize<'de> for VersionedStats {
{
#[derive(Deserialize)]
struct Inner {
#[serde(rename = "5")]
es5: Statistics,
#[serde(rename = "6")]
es6: Statistics,
#[serde(rename = "7")]
es7: Statistics,
#[serde(rename = "8")]
es8: Statistics,
#[serde(rename = "9")]
es9: Statistics,
#[serde(rename = "10")]
es10: Statistics,
#[serde(rename = "11")]
es11: Statistics,
#[serde(rename = "12")]
es12: Statistics,
#[serde(rename = "13")]
es13: Statistics,
#[serde(default)]
#[serde(rename = "14")]
es14: Option<Statistics>,
}

Expand Down Expand Up @@ -740,35 +760,28 @@ impl AddAssign for VersionedStats {
/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "a")]
stats: Statistics,
#[serde(rename = "av", default)]
#[serde(rename = "v", default)]
versioned_stats: VersionedStats,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
#[serde(skip_serializing_if = "FxHashMap::is_empty", default)]
#[serde(rename = "s")]
suites: Vec<SuiteResult>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
suites: FxHashMap<Box<str>, SuiteResult>,
#[serde(skip_serializing_if = "FxHashMap::is_empty", default)]
#[serde(rename = "t")]
tests: Vec<TestResult>,
#[serde(skip_serializing_if = "FxHashSet::is_empty", default)]
#[serde(rename = "f")]
features: FxHashSet<String>,
tests: FxHashMap<Box<str>, TestResult>,
}

/// Outcome of a test.
/// Result of a test, including the outcome for strict and non-strict mode.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[allow(dead_code)]
struct TestResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "v", default)]
edition: SpecEdition,
#[serde(skip)]
result_text: Box<str>,
#[serde(rename = "r")]
result: TestOutcomeResult,
#[serde(rename = "s", default)]
strict: Option<TestOutcomeResult>,
#[serde(rename = "r", default)]
no_strict: Option<TestOutcomeResult>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
Expand Down
Loading

0 comments on commit 3a8360b

Please sign in to comment.