Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed conformance reporting #3348

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Fixed conformance reporting #3348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3a8360b
Fixed conformance reporting
Razican Feb 24, 2024
dca72d4
Small optimization
Razican Feb 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 47 additions & 63 deletions tests/tester/src/exec/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use boa_engine::{
};
use colored::Colorize;
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use rustc_hash::FxHashMap;
use std::{cell::RefCell, eprintln, rc::Rc};

use self::js262::WorkerHandles;
Expand All @@ -34,12 +34,12 @@ impl TestSuite {
max_edition: SpecEdition,
optimizer_options: OptimizerOptions,
console: bool,
) -> SuiteResult {
) -> (Box<str>, SuiteResult) {
if verbose != 0 {
println!("Suite {}:", self.path.display());
}

let suites: Vec<_> = if parallel {
let suites: FxHashMap<_, _> = if parallel {
self.suites
.par_iter()
.map(|suite| {
Expand Down Expand Up @@ -69,7 +69,7 @@ impl TestSuite {
.collect()
};

let tests: Vec<_> = if parallel {
let tests: FxHashMap<_, _> = if parallel {
self.tests
.par_iter()
.filter(|test| test.edition <= max_edition)
Expand All @@ -83,11 +83,6 @@ impl TestSuite {
.collect()
};

let mut features = FxHashSet::default();
for test_iter in &*self.tests {
features.extend(test_iter.features.iter().map(ToString::to_string));
}

if verbose != 0 {
println!();
}
Expand All @@ -96,27 +91,29 @@ impl TestSuite {
let mut versioned_stats = VersionedStats::default();
let mut es_next = Statistics::default();

for test in &tests {
match test.result {
TestOutcomeResult::Passed => {
for test in tests.values() {
match (test.strict, test.no_strict) {
(Some(TestOutcomeResult::Passed), None | Some(TestOutcomeResult::Passed))
| (None, Some(TestOutcomeResult::Passed)) => {
versioned_stats.apply(test.edition, |stats| {
stats.passed += 1;
});
es_next.passed += 1;
}
TestOutcomeResult::Ignored => {
(Some(TestOutcomeResult::Ignored), _) | (_, Some(TestOutcomeResult::Ignored)) => {
versioned_stats.apply(test.edition, |stats| {
stats.ignored += 1;
});
es_next.ignored += 1;
}
TestOutcomeResult::Panic => {
(Some(TestOutcomeResult::Panic), _) | (_, Some(TestOutcomeResult::Panic)) => {
versioned_stats.apply(test.edition, |stats| {
stats.panic += 1;
});
es_next.panic += 1;
}
TestOutcomeResult::Failed => {}
(Some(TestOutcomeResult::Failed), _) | (_, Some(TestOutcomeResult::Failed)) => {}
(None, None) => unreachable!("test should have at least one result"),
}
versioned_stats.apply(test.edition, |stats| {
stats.total += 1;
Expand All @@ -125,10 +122,9 @@ impl TestSuite {
}

// Count total tests
for suite in &suites {
for suite in suites.values() {
versioned_stats += suite.versioned_stats;
es_next += suite.stats;
features.extend(suite.features.iter().cloned());
}

if verbose != 0 {
Expand All @@ -149,48 +145,51 @@ impl TestSuite {
(es_next.passed as f64 / es_next.total as f64) * 100.0
);
}
SuiteResult {
name: self.name.clone(),
stats: es_next,
versioned_stats,
suites,
tests,
features,
}
(
self.name.clone(),
SuiteResult {
stats: es_next,
versioned_stats,
suites,
tests,
},
)
}
}

impl Test {
/// Runs the test.
///
/// Returns the test name and the result of the test.
pub(crate) fn run(
&self,
harness: &Harness,
verbose: u8,
optimizer_options: OptimizerOptions,
console: bool,
) -> TestResult {
) -> (Box<str>, TestResult) {
let mut result = TestResult {
edition: self.edition,
strict: None,
no_strict: None,
};

if self.flags.contains(TestFlags::MODULE) || self.flags.contains(TestFlags::RAW) {
return self.run_once(harness, false, verbose, optimizer_options, console);
}
result.no_strict =
Some(self.run_once(harness, false, verbose, optimizer_options, console));
} else {
if self.flags.contains(TestFlags::STRICT) {
result.strict =
Some(self.run_once(harness, true, verbose, optimizer_options, console));
}

if self
.flags
.contains(TestFlags::STRICT | TestFlags::NO_STRICT)
{
let r = self.run_once(harness, false, verbose, optimizer_options, console);
if r.result != TestOutcomeResult::Passed {
return r;
if self.flags.contains(TestFlags::NO_STRICT) {
result.no_strict =
Some(self.run_once(harness, false, verbose, optimizer_options, console));
}
self.run_once(harness, true, verbose, optimizer_options, console)
} else {
self.run_once(
harness,
self.flags.contains(TestFlags::STRICT),
verbose,
optimizer_options,
console,
)
}

(self.name.clone(), result)
}

/// Runs the test once, in strict or non-strict mode
Expand All @@ -201,7 +200,7 @@ impl Test {
verbose: u8,
optimizer_options: OptimizerOptions,
console: bool,
) -> TestResult {
) -> TestOutcomeResult {
let Ok(source) = Source::from_filepath(&self.path) else {
if verbose > 1 {
println!(
Expand All @@ -213,12 +212,7 @@ impl Test {
} else {
print!("{}", "F".red());
}
return TestResult {
name: self.name.clone(),
edition: self.edition,
result: TestOutcomeResult::Failed,
result_text: Box::from("Could not read test file."),
};
return TestOutcomeResult::Failed;
};
if self.ignored {
if verbose > 1 {
Expand All @@ -231,12 +225,7 @@ impl Test {
} else {
print!("{}", "-".yellow());
}
return TestResult {
name: self.name.clone(),
edition: self.edition,
result: TestOutcomeResult::Ignored,
result_text: Box::default(),
};
return TestOutcomeResult::Ignored;
}
if verbose > 1 {
println!(
Expand Down Expand Up @@ -582,12 +571,7 @@ impl Test {
println!();
}

TestResult {
name: self.name.clone(),
edition: self.edition,
result,
result_text: result_text.into_boxed_str(),
}
result
}

/// Sets the environment up to run the test.
Expand Down
49 changes: 31 additions & 18 deletions tests/tester/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ fn run_test_suite(
if verbose != 0 {
println!("Test suite loaded, starting tests...");
}
let results = suite.run(
let (_name, results) = suite.run(
&harness,
verbose,
parallel,
Expand Down Expand Up @@ -590,15 +590,25 @@ impl AddAssign for Statistics {
/// Represents tests statistics separated by ECMAScript edition
#[derive(Default, Debug, Copy, Clone, Serialize)]
struct VersionedStats {
#[serde(rename = "5")]
es5: Statistics,
#[serde(rename = "6")]
es6: Statistics,
#[serde(rename = "7")]
es7: Statistics,
#[serde(rename = "8")]
es8: Statistics,
#[serde(rename = "9")]
es9: Statistics,
#[serde(rename = "10")]
es10: Statistics,
#[serde(rename = "11")]
es11: Statistics,
#[serde(rename = "12")]
es12: Statistics,
#[serde(rename = "13")]
es13: Statistics,
#[serde(rename = "14")]
es14: Statistics,
}

Expand All @@ -609,16 +619,26 @@ impl<'de> Deserialize<'de> for VersionedStats {
{
#[derive(Deserialize)]
struct Inner {
#[serde(rename = "5")]
es5: Statistics,
#[serde(rename = "6")]
es6: Statistics,
#[serde(rename = "7")]
es7: Statistics,
#[serde(rename = "8")]
es8: Statistics,
#[serde(rename = "9")]
es9: Statistics,
#[serde(rename = "10")]
es10: Statistics,
#[serde(rename = "11")]
es11: Statistics,
#[serde(rename = "12")]
es12: Statistics,
#[serde(rename = "13")]
es13: Statistics,
#[serde(default)]
#[serde(rename = "14")]
es14: Option<Statistics>,
}

Expand Down Expand Up @@ -740,35 +760,28 @@ impl AddAssign for VersionedStats {
/// Outcome of a test suite.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SuiteResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "a")]
stats: Statistics,
#[serde(rename = "av", default)]
#[serde(rename = "v", default)]
versioned_stats: VersionedStats,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
#[serde(skip_serializing_if = "FxHashMap::is_empty", default)]
#[serde(rename = "s")]
suites: Vec<SuiteResult>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
suites: FxHashMap<Box<str>, SuiteResult>,
#[serde(skip_serializing_if = "FxHashMap::is_empty", default)]
#[serde(rename = "t")]
tests: Vec<TestResult>,
#[serde(skip_serializing_if = "FxHashSet::is_empty", default)]
#[serde(rename = "f")]
features: FxHashSet<String>,
tests: FxHashMap<Box<str>, TestResult>,
}

/// Outcome of a test.
/// Result of a test, including the outcome for strict and non-strict mode.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[allow(dead_code)]
struct TestResult {
#[serde(rename = "n")]
name: Box<str>,
#[serde(rename = "v", default)]
edition: SpecEdition,
#[serde(skip)]
result_text: Box<str>,
#[serde(rename = "r")]
result: TestOutcomeResult,
#[serde(rename = "s", default)]
strict: Option<TestOutcomeResult>,
#[serde(rename = "r", default)]
no_strict: Option<TestOutcomeResult>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
Expand Down
Loading
Loading