Skip to content

Commit 3a8360b

Browse files
RazicanRazican
committed
Fixed conformance reporting
1 parent c94e10d commit 3a8360b

File tree

3 files changed

+177
-190
lines changed

3 files changed

+177
-190
lines changed

‎tests/tester/src/exec/mod.rs‎

Lines changed: 47 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use boa_engine::{
1919
};
2020
use colored::Colorize;
2121
use rayon::prelude::*;
22-
use rustc_hash::FxHashSet;
22+
use rustc_hash::FxHashMap;
2323
use std::{cell::RefCell, eprintln, rc::Rc};
2424

2525
use self::js262::WorkerHandles;
@@ -34,12 +34,12 @@ impl TestSuite {
3434
max_edition: SpecEdition,
3535
optimizer_options: OptimizerOptions,
3636
console: bool,
37-
) -> SuiteResult {
37+
) -> (Box<str>, SuiteResult) {
3838
if verbose != 0 {
3939
println!("Suite {}:", self.path.display());
4040
}
4141

42-
let suites: Vec<_> = if parallel {
42+
let suites: FxHashMap<_, _> = if parallel {
4343
self.suites
4444
.par_iter()
4545
.map(|suite| {
@@ -69,7 +69,7 @@ impl TestSuite {
6969
.collect()
7070
};
7171

72-
let tests: Vec<_> = if parallel {
72+
let tests: FxHashMap<_, _> = if parallel {
7373
self.tests
7474
.par_iter()
7575
.filter(|test| test.edition <= max_edition)
@@ -83,11 +83,6 @@ impl TestSuite {
8383
.collect()
8484
};
8585

86-
let mut features = FxHashSet::default();
87-
for test_iter in &*self.tests {
88-
features.extend(test_iter.features.iter().map(ToString::to_string));
89-
}
90-
9186
if verbose != 0 {
9287
println!();
9388
}
@@ -96,27 +91,29 @@ impl TestSuite {
9691
let mut versioned_stats = VersionedStats::default();
9792
let mut es_next = Statistics::default();
9893

99-
for test in &tests {
100-
match test.result {
101-
TestOutcomeResult::Passed => {
94+
for test in tests.values() {
95+
match (test.strict, test.no_strict) {
96+
(Some(TestOutcomeResult::Passed), None | Some(TestOutcomeResult::Passed))
97+
| (None, Some(TestOutcomeResult::Passed)) => {
10298
versioned_stats.apply(test.edition, |stats| {
10399
stats.passed += 1;
104100
});
105101
es_next.passed += 1;
106102
}
107-
TestOutcomeResult::Ignored => {
103+
(Some(TestOutcomeResult::Ignored), _) | (_, Some(TestOutcomeResult::Ignored)) => {
108104
versioned_stats.apply(test.edition, |stats| {
109105
stats.ignored += 1;
110106
});
111107
es_next.ignored += 1;
112108
}
113-
TestOutcomeResult::Panic => {
109+
(Some(TestOutcomeResult::Panic), _) | (_, Some(TestOutcomeResult::Panic)) => {
114110
versioned_stats.apply(test.edition, |stats| {
115111
stats.panic += 1;
116112
});
117113
es_next.panic += 1;
118114
}
119-
TestOutcomeResult::Failed => {}
115+
(Some(TestOutcomeResult::Failed), _) | (_, Some(TestOutcomeResult::Failed)) => {}
116+
(None, None) => unreachable!("test should have at least one result"),
120117
}
121118
versioned_stats.apply(test.edition, |stats| {
122119
stats.total += 1;
@@ -125,10 +122,9 @@ impl TestSuite {
125122
}
126123

127124
// Count total tests
128-
for suite in &suites {
125+
for suite in suites.values() {
129126
versioned_stats += suite.versioned_stats;
130127
es_next += suite.stats;
131-
features.extend(suite.features.iter().cloned());
132128
}
133129

134130
if verbose != 0 {
@@ -149,48 +145,51 @@ impl TestSuite {
149145
(es_next.passed as f64 / es_next.total as f64) * 100.0
150146
);
151147
}
152-
SuiteResult {
153-
name: self.name.clone(),
154-
stats: es_next,
155-
versioned_stats,
156-
suites,
157-
tests,
158-
features,
159-
}
148+
(
149+
self.name.clone(),
150+
SuiteResult {
151+
stats: es_next,
152+
versioned_stats,
153+
suites,
154+
tests,
155+
},
156+
)
160157
}
161158
}
162159

163160
impl Test {
164161
/// Runs the test.
162+
///
163+
/// Returns the test name and the result of the test.
165164
pub(crate) fn run(
166165
&self,
167166
harness: &Harness,
168167
verbose: u8,
169168
optimizer_options: OptimizerOptions,
170169
console: bool,
171-
) -> TestResult {
170+
) -> (Box<str>, TestResult) {
171+
let mut result = TestResult {
172+
edition: self.edition,
173+
strict: None,
174+
no_strict: None,
175+
};
176+
172177
if self.flags.contains(TestFlags::MODULE) || self.flags.contains(TestFlags::RAW) {
173-
return self.run_once(harness, false, verbose, optimizer_options, console);
174-
}
178+
result.no_strict =
179+
Some(self.run_once(harness, false, verbose, optimizer_options, console));
180+
} else {
181+
if self.flags.contains(TestFlags::STRICT) && !self.flags.contains(TestFlags::RAW) {
182+
result.strict =
183+
Some(self.run_once(harness, true, verbose, optimizer_options, console));
184+
}
175185

176-
if self
177-
.flags
178-
.contains(TestFlags::STRICT | TestFlags::NO_STRICT)
179-
{
180-
let r = self.run_once(harness, false, verbose, optimizer_options, console);
181-
if r.result != TestOutcomeResult::Passed {
182-
return r;
186+
if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) {
187+
result.no_strict =
188+
Some(self.run_once(harness, false, verbose, optimizer_options, console));
183189
}
184-
self.run_once(harness, true, verbose, optimizer_options, console)
185-
} else {
186-
self.run_once(
187-
harness,
188-
self.flags.contains(TestFlags::STRICT),
189-
verbose,
190-
optimizer_options,
191-
console,
192-
)
193190
}
191+
192+
(self.name.clone(), result)
194193
}
195194

196195
/// Runs the test once, in strict or non-strict mode
@@ -201,7 +200,7 @@ impl Test {
201200
verbose: u8,
202201
optimizer_options: OptimizerOptions,
203202
console: bool,
204-
) -> TestResult {
203+
) -> TestOutcomeResult {
205204
let Ok(source) = Source::from_filepath(&self.path) else {
206205
if verbose > 1 {
207206
println!(
@@ -213,12 +212,7 @@ impl Test {
213212
} else {
214213
print!("{}", "F".red());
215214
}
216-
return TestResult {
217-
name: self.name.clone(),
218-
edition: self.edition,
219-
result: TestOutcomeResult::Failed,
220-
result_text: Box::from("Could not read test file."),
221-
};
215+
return TestOutcomeResult::Failed;
222216
};
223217
if self.ignored {
224218
if verbose > 1 {
@@ -231,12 +225,7 @@ impl Test {
231225
} else {
232226
print!("{}", "-".yellow());
233227
}
234-
return TestResult {
235-
name: self.name.clone(),
236-
edition: self.edition,
237-
result: TestOutcomeResult::Ignored,
238-
result_text: Box::default(),
239-
};
228+
return TestOutcomeResult::Ignored;
240229
}
241230
if verbose > 1 {
242231
println!(
@@ -582,12 +571,7 @@ impl Test {
582571
println!();
583572
}
584573

585-
TestResult {
586-
name: self.name.clone(),
587-
edition: self.edition,
588-
result,
589-
result_text: result_text.into_boxed_str(),
590-
}
574+
result
591575
}
592576

593577
/// Sets the environment up to run the test.

‎tests/tester/src/main.rs‎

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ fn run_test_suite(
451451
if verbose != 0 {
452452
println!("Test suite loaded, starting tests...");
453453
}
454-
let results = suite.run(
454+
let (_name, results) = suite.run(
455455
&harness,
456456
verbose,
457457
parallel,
@@ -590,15 +590,25 @@ impl AddAssign for Statistics {
590590
/// Represents tests statistics separated by ECMAScript edition
591591
#[derive(Default, Debug, Copy, Clone, Serialize)]
592592
struct VersionedStats {
593+
#[serde(rename = "5")]
593594
es5: Statistics,
595+
#[serde(rename = "6")]
594596
es6: Statistics,
597+
#[serde(rename = "7")]
595598
es7: Statistics,
599+
#[serde(rename = "8")]
596600
es8: Statistics,
601+
#[serde(rename = "9")]
597602
es9: Statistics,
603+
#[serde(rename = "10")]
598604
es10: Statistics,
605+
#[serde(rename = "11")]
599606
es11: Statistics,
607+
#[serde(rename = "12")]
600608
es12: Statistics,
609+
#[serde(rename = "13")]
601610
es13: Statistics,
611+
#[serde(rename = "14")]
602612
es14: Statistics,
603613
}
604614

@@ -609,16 +619,26 @@ impl<'de> Deserialize<'de> for VersionedStats {
609619
{
610620
#[derive(Deserialize)]
611621
struct Inner {
622+
#[serde(rename = "5")]
612623
es5: Statistics,
624+
#[serde(rename = "6")]
613625
es6: Statistics,
626+
#[serde(rename = "7")]
614627
es7: Statistics,
628+
#[serde(rename = "8")]
615629
es8: Statistics,
630+
#[serde(rename = "9")]
616631
es9: Statistics,
632+
#[serde(rename = "10")]
617633
es10: Statistics,
634+
#[serde(rename = "11")]
618635
es11: Statistics,
636+
#[serde(rename = "12")]
619637
es12: Statistics,
638+
#[serde(rename = "13")]
620639
es13: Statistics,
621640
#[serde(default)]
641+
#[serde(rename = "14")]
622642
es14: Option<Statistics>,
623643
}
624644

@@ -740,35 +760,28 @@ impl AddAssign for VersionedStats {
740760
/// Outcome of a test suite.
741761
#[derive(Debug, Clone, Serialize, Deserialize)]
742762
struct SuiteResult {
743-
#[serde(rename = "n")]
744-
name: Box<str>,
745763
#[serde(rename = "a")]
746764
stats: Statistics,
747-
#[serde(rename = "av", default)]
765+
#[serde(rename = "v", default)]
748766
versioned_stats: VersionedStats,
749-
#[serde(skip_serializing_if = "Vec::is_empty", default)]
767+
#[serde(skip_serializing_if = "FxHashMap::is_empty", default)]
750768
#[serde(rename = "s")]
751-
suites: Vec<SuiteResult>,
752-
#[serde(skip_serializing_if = "Vec::is_empty", default)]
769+
suites: FxHashMap<Box<str>, SuiteResult>,
770+
#[serde(skip_serializing_if = "FxHashMap::is_empty", default)]
753771
#[serde(rename = "t")]
754-
tests: Vec<TestResult>,
755-
#[serde(skip_serializing_if = "FxHashSet::is_empty", default)]
756-
#[serde(rename = "f")]
757-
features: FxHashSet<String>,
772+
tests: FxHashMap<Box<str>, TestResult>,
758773
}
759774

760-
/// Outcome of a test.
775+
/// Result of a test, including the outcome for strict and non-strict mode.
761776
#[derive(Debug, Clone, Serialize, Deserialize)]
762777
#[allow(dead_code)]
763778
struct TestResult {
764-
#[serde(rename = "n")]
765-
name: Box<str>,
766779
#[serde(rename = "v", default)]
767780
edition: SpecEdition,
768-
#[serde(skip)]
769-
result_text: Box<str>,
770-
#[serde(rename = "r")]
771-
result: TestOutcomeResult,
781+
#[serde(rename = "s", default)]
782+
strict: Option<TestOutcomeResult>,
783+
#[serde(rename = "r", default)]
784+
no_strict: Option<TestOutcomeResult>,
772785
}
773786

774787
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]

0 commit comments

Comments
 (0)