diff --git a/.gitignore b/.gitignore index 040db13443d..e446fa4a7c7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ yarn-error.log .boa_history # test262 testing suite -test262 +/test262 # Profiling *.string_data diff --git a/Cargo.lock b/Cargo.lock index 54ef56293e1..443ecdb99a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,21 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "assert_cmd" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -555,7 +570,7 @@ dependencies = [ name = "boa_tester" version = "0.19.0" dependencies = [ - "bitflags 2.6.0", + "assert_cmd", "boa_engine", "boa_gc", "boa_runtime", @@ -564,13 +579,11 @@ dependencies = [ "color-eyre", "colored", "comfy-table", - "phf", "rayon", "rustc-hash 2.0.0", "serde", "serde_json", - "serde_repr", - "serde_yaml", + "test262", "time", "toml 0.8.19", ] @@ -586,6 +599,17 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +dependencies = [ + "memchr", + "regex-automata 0.4.7", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -1097,6 +1121,12 @@ dependencies = [ "thousands", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "displaydoc" version = "0.2.5" @@ -1108,6 +1138,12 @@ dependencies = [ "syn", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -2630,6 +2666,33 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -2994,17 +3057,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_repr" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "serde_spanned" version = "0.6.7" @@ -3240,6 +3292,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "test-case" version = "3.3.1" @@ -3273,6 +3331,19 @@ dependencies = [ "test-case-core", ] +[[package]] +name = "test262" +version = "0.19.0" +dependencies = [ + "bitflags 2.6.0", + "color-eyre", + "phf", + "rustc-hash 2.0.0", + "serde", + "serde_json", + "serde_yaml", +] + [[package]] name = "textwrap" version = "0.16.1" @@ -3655,6 +3726,15 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.2.0" diff --git a/tests/tester/Cargo.toml b/tests/tester/Cargo.toml index 0216531282c..3c2156f32dc 100644 --- a/tests/tester/Cargo.toml +++ b/tests/tester/Cargo.toml @@ -17,20 +17,20 @@ boa_runtime.workspace = true boa_gc.workspace = true clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } -serde_yaml = "0.9.34" # TODO: Track https://github.com/saphyr-rs/saphyr. +test262 = { path = "../../tools/test262"} serde_json.workspace = true -bitflags.workspace = true colored.workspace = true rustc-hash = { workspace = true, features = ["std"] } rayon.workspace = true toml.workspace = true color-eyre.workspace = true -phf = { workspace = true, features = ["macros"] } comfy-table.workspace = true -serde_repr.workspace = true bus.workspace = true time.workspace = true +[dev-dependencies] +assert_cmd = "2.0.14" + [features] default = ["boa_engine/intl_bundled", "boa_engine/experimental", "boa_engine/annex-b"] diff --git a/tests/tester/src/exec/mod.rs b/tests/tester/src/exec/mod.rs index e5ce0553cfd..bb9277a4194 100644 --- a/tests/tester/src/exec/mod.rs +++ b/tests/tester/src/exec/mod.rs @@ -2,10 +2,7 @@ mod js262; -use crate::{ - read::ErrorType, Harness, Outcome, Phase, SpecEdition, Statistics, SuiteResult, Test, - TestFlags, TestOutcomeResult, TestResult, TestSuite, VersionedStats, -}; +use crate::{Statistics, SuiteResult, TestFlags, TestOutcomeResult, TestResult, VersionedStats}; use boa_engine::{ builtins::promise::PromiseState, js_str, js_string, @@ -22,12 +19,41 @@ use colored::Colorize; use rayon::prelude::*; use rustc_hash::FxHashSet; use std::{cell::RefCell, eprintln, path::Path, rc::Rc}; +use test262::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSuite}; use self::js262::WorkerHandles; -impl TestSuite { +pub(crate) trait RunTest { + /// Runs the test. + fn run(&self, harness: &Harness, verbose: u8, optimizer_options: OptimizerOptions, console: bool) -> TestResult; + + /// Runs the test once, in strict or non-strict mode + fn run_once( + &self, + harness: &Harness, + strict: bool, + verbose: u8, + optimizer_options: OptimizerOptions, + console: bool, + ) -> TestResult; +} + +pub(crate) trait RunTestSuite{ + /// Runs the test suite. + fn run( + &self, + harness: &Harness, + verbose: u8, + parallel: bool, + max_edition: SpecEdition, + optimizer_options: OptimizerOptions, + console: bool, + ) -> SuiteResult; +} + +impl RunTestSuite for TestSuite { /// Runs the test suite. - pub(crate) fn run( + fn run( &self, harness: &Harness, verbose: u8, @@ -161,9 +187,9 @@ impl TestSuite { } } -impl Test { +impl RunTest for Test { /// Runs the test. - pub(crate) fn run( + fn run( &self, harness: &Harness, verbose: u8, @@ -194,54 +220,6 @@ impl Test { } } - /// Creates the test result from the outcome and message. - fn create_result>>( - &self, - outcome: TestOutcomeResult, - text: S, - strict: bool, - verbosity: u8, - ) -> TestResult { - let result_text = text.into(); - - if verbosity > 1 { - println!( - "`{}`{}: {}", - self.path.display(), - if strict { " (strict)" } else { "" }, - match outcome { - TestOutcomeResult::Passed => "Passed".green(), - TestOutcomeResult::Ignored => "Ignored".yellow(), - TestOutcomeResult::Failed => "Failed".red(), - TestOutcomeResult::Panic => "⚠ Panic ⚠".red(), - } - ); - } else { - let symbol = match outcome { - TestOutcomeResult::Passed => ".".green(), - TestOutcomeResult::Ignored => "-".yellow(), - TestOutcomeResult::Failed | TestOutcomeResult::Panic => "F".red(), - }; - - print!("{symbol}"); - } - - if verbosity > 2 { - println!( - "`{}`{}: result text\n{result_text}\n", - self.path.display(), - if strict { " (strict)" } else { "" }, - ); - } - - TestResult { - name: self.name.clone(), - edition: self.edition, - result_text, - result: outcome, - } - } - /// Runs the test once, in strict or non-strict mode fn run_once( &self, @@ -252,7 +230,10 @@ impl Test { console: bool, ) -> TestResult { let Ok(source) = Source::from_filepath(&self.path) else { - return self.create_result( + return create_result( + &self.path, + &self.name, + &self.edition, TestOutcomeResult::Failed, "Could not read test file", strict, @@ -261,7 +242,7 @@ impl Test { }; if self.ignored { - return self.create_result(TestOutcomeResult::Ignored, "", strict, verbosity); + return create_result(&self.path, &self.name, &self.edition, TestOutcomeResult::Ignored, "", strict, verbosity); } if verbosity > 1 { @@ -275,7 +256,7 @@ impl Test { let result = std::panic::catch_unwind(|| match self.expected_outcome { Outcome::Positive => { let (ref mut context, async_result, mut handles) = - match self.create_context(harness, optimizer_options, console) { + match create_context(&self.path, &self.flags, &self.includes, harness, optimizer_options, console) { Ok(r) => r, Err(e) => return (false, e), }; @@ -375,7 +356,7 @@ impl Test { phase: Phase::Resolution, error_type, } => { - let context = &mut match self.create_context(harness, optimizer_options, console) { + let context = &mut match create_context(&self.path, &self.flags, &self.includes, harness, optimizer_options, console) { Ok(r) => r, Err(e) => return (false, e), } @@ -420,7 +401,7 @@ impl Test { error_type, } => { let (ref mut context, _async_result, mut handles) = - match self.create_context(harness, optimizer_options, console) { + match create_context(&self.path, &self.flags, &self.includes, harness, optimizer_options, console) { Ok(r) => r, Err(e) => return (false, e), }; @@ -502,82 +483,134 @@ impl Test { }, ); - self.create_result(result, result_text, strict, verbosity) + create_result(&self.path, &self.name, &self.edition, result, result_text, strict, verbosity) } +} - /// Creates the context to run the test. - fn create_context( - &self, - harness: &Harness, - optimizer_options: OptimizerOptions, - console: bool, - ) -> Result<(Context, AsyncResult, WorkerHandles), String> { - let async_result = AsyncResult::default(); - let handles = WorkerHandles::new(); - let loader = Rc::new( - SimpleModuleLoader::new(self.path.parent().expect("test should have a parent dir")) - .expect("test path should be canonicalizable"), - ); - let mut context = Context::builder() - .module_loader(loader.clone()) - .can_block(!self.flags.contains(TestFlags::CAN_BLOCK_IS_FALSE)) - .build() - .expect("cannot fail with default global object"); - - context.set_optimizer_options(optimizer_options); - - // Register the print() function. - register_print_fn(&mut context, async_result.clone()); +/// Creates the context to run the test. +fn create_context( + path: &Path, + flags: &TestFlags, + includes: &FxHashSet>, + harness: &Harness, + optimizer_options: OptimizerOptions, + console: bool, +) -> Result<(Context, AsyncResult, WorkerHandles), String> { + let async_result = AsyncResult::default(); + let handles = WorkerHandles::new(); + let loader = Rc::new( + SimpleModuleLoader::new(path.parent().expect("test should have a parent dir")) + .expect("test path should be canonicalizable"), + ); + let mut context = Context::builder() + .module_loader(loader.clone()) + .can_block(!flags.contains(TestFlags::CAN_BLOCK_IS_FALSE)) + .build() + .expect("cannot fail with default global object"); + + context.set_optimizer_options(optimizer_options); + + // Register the print() function. + register_print_fn(&mut context, async_result.clone()); + + // add the $262 object. + let _js262 = js262::register_js262(handles.clone(), &mut context); + + if console { + let console = boa_runtime::Console::init(&mut context); + context + .register_global_property(boa_runtime::Console::NAME, console, Attribute::all()) + .expect("the console builtin shouldn't exist"); + } - // add the $262 object. - let _js262 = js262::register_js262(handles.clone(), &mut context); + if flags.contains(TestFlags::RAW) { + return Ok((context, async_result, handles)); + } - if console { - let console = boa_runtime::Console::init(&mut context); - context - .register_global_property(boa_runtime::Console::NAME, console, Attribute::all()) - .expect("the console builtin shouldn't exist"); - } + let assert = Source::from_reader( + harness.assert.content.as_bytes(), + Some(&harness.assert.path), + ); + let sta = Source::from_reader(harness.sta.content.as_bytes(), Some(&harness.sta.path)); - if self.flags.contains(TestFlags::RAW) { - return Ok((context, async_result, handles)); - } + context + .eval(assert) + .map_err(|e| format!("could not run assert.js:\n{e}"))?; + context + .eval(sta) + .map_err(|e| format!("could not run sta.js:\n{e}"))?; - let assert = Source::from_reader( - harness.assert.content.as_bytes(), - Some(&harness.assert.path), + if flags.contains(TestFlags::ASYNC) { + let dph = Source::from_reader( + harness.doneprint_handle.content.as_bytes(), + Some(&harness.doneprint_handle.path), ); - let sta = Source::from_reader(harness.sta.content.as_bytes(), Some(&harness.sta.path)); - context - .eval(assert) - .map_err(|e| format!("could not run assert.js:\n{e}"))?; + .eval(dph) + .map_err(|e| format!("could not run doneprintHandle.js:\n{e}"))?; + } + + for include_name in includes { + let include = harness + .includes + .get(include_name) + .ok_or_else(|| format!("could not find the {include_name} include file."))?; + let source = Source::from_reader(include.content.as_bytes(), Some(&include.path)); context - .eval(sta) - .map_err(|e| format!("could not run sta.js:\n{e}"))?; + .eval(source) + .map_err(|e| format!("could not run the harness `{include_name}`:\nUncaught {e}",))?; + } - if self.flags.contains(TestFlags::ASYNC) { - let dph = Source::from_reader( - harness.doneprint_handle.content.as_bytes(), - Some(&harness.doneprint_handle.path), - ); - context - .eval(dph) - .map_err(|e| format!("could not run doneprintHandle.js:\n{e}"))?; - } + Ok((context, async_result, handles)) +} - for include_name in &self.includes { - let include = harness - .includes - .get(include_name) - .ok_or_else(|| format!("could not find the {include_name} include file."))?; - let source = Source::from_reader(include.content.as_bytes(), Some(&include.path)); - context.eval(source).map_err(|e| { - format!("could not run the harness `{include_name}`:\nUncaught {e}",) - })?; - } +/// Creates the test result from the outcome and message. +fn create_result>>( + path: &Path, + name: &Box, + edition: &SpecEdition, + outcome: TestOutcomeResult, + text: S, + strict: bool, + verbosity: u8, +) -> TestResult { + let result_text = text.into(); + + if verbosity > 1 { + println!( + "`{}`{}: {}", + path.display(), + if strict { " (strict)" } else { "" }, + match outcome { + TestOutcomeResult::Passed => "Passed".green(), + TestOutcomeResult::Ignored => "Ignored".yellow(), + TestOutcomeResult::Failed => "Failed".red(), + TestOutcomeResult::Panic => "⚠ Panic ⚠".red(), + } + ); + } else { + let symbol = match outcome { + TestOutcomeResult::Passed => ".".green(), + TestOutcomeResult::Ignored => "-".yellow(), + TestOutcomeResult::Failed | TestOutcomeResult::Panic => "F".red(), + }; + + print!("{symbol}"); + } + + if verbosity > 2 { + println!( + "`{}`{}: result text\n{result_text}\n", + path.display(), + if strict { " (strict)" } else { "" }, + ); + } - Ok((context, async_result, handles)) + TestResult { + name: name.clone(), + edition: edition.clone(), + result_text, + result: outcome, } } diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index bc2b8624833..6268355217a 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -11,40 +11,31 @@ clippy::print_stdout )] +mod exec; +mod results; + +use exec::{RunTest, RunTestSuite}; +use test262::{Harness, Ignored, SpecEdition, TestFlags}; + +use self::results::{compare_results, write_json}; + +use boa_engine::optimizer::OptimizerOptions; use std::{ ops::{Add, AddAssign}, path::{Path, PathBuf}, - process::Command, sync::OnceLock, time::Instant, }; -use bitflags::bitflags; use clap::{ArgAction, Parser, ValueHint}; use color_eyre::{ - eyre::{bail, eyre, WrapErr}, + eyre::eyre, + eyre::{bail, WrapErr}, Result, }; use colored::Colorize; -use rustc_hash::{FxHashMap, FxHashSet}; -use serde::{ - de::{Unexpected, Visitor}, - Deserialize, Deserializer, Serialize, -}; - -use boa_engine::optimizer::OptimizerOptions; -use edition::SpecEdition; -use read::ErrorType; - -use self::{ - read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag}, - results::{compare_results, write_json}, -}; - -mod edition; -mod exec; -mod read; -mod results; +use rustc_hash::FxHashSet; +use serde::{Deserialize, Deserializer, Serialize}; static START: OnceLock = OnceLock::new(); @@ -69,54 +60,6 @@ impl Config { } } -/// Structure to allow defining ignored tests, features and files that should -/// be ignored even when reading. -#[derive(Debug, Deserialize)] -struct Ignored { - #[serde(default)] - tests: FxHashSet>, - #[serde(default)] - features: FxHashSet>, - #[serde(default = "TestFlags::empty")] - flags: TestFlags, -} - -impl Ignored { - /// Checks if the ignore list contains the given test name in the list of - /// tests to ignore. - pub(crate) fn contains_test(&self, test: &str) -> bool { - self.tests.contains(test) - } - - /// Checks if the ignore list contains the given feature name in the list - /// of features to ignore. - pub(crate) fn contains_feature(&self, feature: &str) -> bool { - if self.features.contains(feature) { - return true; - } - // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. - // This ensures those are also ignored. - feature - .split('.') - .next() - .is_some_and(|feat| self.features.contains(feat)) - } - - pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool { - flags.intersects(self.flags) - } -} - -impl Default for Ignored { - fn default() -> Self { - Self { - tests: FxHashSet::default(), - features: FxHashSet::default(), - flags: TestFlags::empty(), - } - } -} - /// Boa test262 tester #[derive(Debug, Parser)] #[command(author, version, about, name = "Boa test262 tester")] @@ -187,8 +130,6 @@ enum Cli { }, } -const DEFAULT_TEST262_DIRECTORY: &str = "test262"; - /// Program entry point. fn main() -> Result<()> { color_eyre::install()?; @@ -234,9 +175,9 @@ fn main() -> Result<()> { let test262_path = if let Some(path) = test262_path.as_deref() { path } else { - clone_test262(test262_commit, verbose)?; - - Path::new(DEFAULT_TEST262_DIRECTORY) + // TODO remove test262 fn (unused) + // test262::clone_test262(test262_commit, verbose)?; + Path::new(test262::TEST262_DIRECTORY) } .canonicalize(); let test262_path = &test262_path.wrap_err("could not get the Test262 path")?; @@ -245,6 +186,7 @@ fn main() -> Result<()> { &config, verbose, !disable_parallelism, + test262_commit, test262_path, suite.as_path(), output.as_deref(), @@ -266,148 +208,13 @@ fn main() -> Result<()> { } } -/// Returns the commit hash and commit message of the provided branch name. -fn get_last_branch_commit(branch: &str, verbose: u8) -> Result<(String, String)> { - if verbose > 1 { - println!("Getting last commit on '{branch}' branch"); - } - let result = Command::new("git") - .arg("log") - .args(["-n", "1"]) - .arg("--pretty=format:%H %s") - .arg(branch) - .current_dir(DEFAULT_TEST262_DIRECTORY) - .output()?; - - if !result.status.success() { - bail!( - "test262 getting commit hash and message failed with return code {:?}", - result.status.code() - ); - } - - let output = std::str::from_utf8(&result.stdout)?.trim(); - - let (hash, message) = output - .split_once(' ') - .expect("git log output to contain hash and message"); - - Ok((hash.into(), message.into())) -} - -fn reset_test262_commit(commit: &str, verbose: u8) -> Result<()> { - if verbose != 0 { - println!("Reset test262 to commit: {commit}..."); - } - - let result = Command::new("git") - .arg("reset") - .arg("--hard") - .arg(commit) - .current_dir(DEFAULT_TEST262_DIRECTORY) - .status()?; - - if !result.success() { - bail!( - "test262 commit {commit} checkout failed with return code: {:?}", - result.code() - ); - } - - Ok(()) -} - -fn clone_test262(commit: Option<&str>, verbose: u8) -> Result<()> { - const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; - - let update = commit.is_none(); - - if Path::new(DEFAULT_TEST262_DIRECTORY).is_dir() { - let (current_commit_hash, current_commit_message) = - get_last_branch_commit("HEAD", verbose)?; - - if let Some(commit) = commit { - if current_commit_hash == commit { - return Ok(()); - } - } - - if verbose != 0 { - println!("Fetching latest test262 commits..."); - } - let result = Command::new("git") - .arg("fetch") - .current_dir(DEFAULT_TEST262_DIRECTORY) - .status()?; - - if !result.success() { - bail!( - "Test262 fetching latest failed with return code {:?}", - result.code() - ); - } - - if let Some(commit) = commit { - println!("Test262 switching to commit {commit}..."); - reset_test262_commit(commit, verbose)?; - return Ok(()); - } - - if verbose != 0 { - println!("Checking latest Test262 with current HEAD..."); - } - let (latest_commit_hash, latest_commit_message) = - get_last_branch_commit("origin/main", verbose)?; - - if current_commit_hash != latest_commit_hash { - if update { - println!("Updating Test262 repository:"); - } else { - println!("Warning Test262 repository is not in sync, use '--test262-commit latest' to automatically update it:"); - } - - println!(" Current commit: {current_commit_hash} {current_commit_message}"); - println!(" Latest commit: {latest_commit_hash} {latest_commit_message}"); - - if update { - reset_test262_commit(&latest_commit_hash, verbose)?; - } - } - - return Ok(()); - } - - println!("Cloning test262..."); - let result = Command::new("git") - .arg("clone") - .arg(TEST262_REPOSITORY) - .arg(DEFAULT_TEST262_DIRECTORY) - .status()?; - - if !result.success() { - bail!( - "Cloning Test262 repository failed with return code {:?}", - result.code() - ); - } - - if let Some(commit) = commit { - if verbose != 0 { - println!("Reset Test262 to commit: {commit}..."); - } - - reset_test262_commit(commit, verbose)?; - } - - Ok(()) -} - /// Runs the full test suite. #[allow(clippy::too_many_arguments)] fn run_test_suite( config: &Config, verbose: u8, parallel: bool, + test262_commit: Option<&str>, test262_path: &Path, suite: &Path, output: Option<&Path>, @@ -429,135 +236,118 @@ fn run_test_suite( if verbose != 0 { println!("Loading the test suite..."); } - let harness = read_harness(test262_path).wrap_err("could not read harness")?; - if suite.to_string_lossy().ends_with(".js") { - let test = read_test(&test262_path.join(suite)).wrap_err_with(|| { - let suite = suite.display(); - format!("could not read the test {suite}") - })?; + let result = test262::read( + suite.to_owned(), + test262::ReadOptions { + test262_path: test262_path.to_owned(), + test262_commit, // TODO - Some(config.commit().to_string()) - priority config/arg?? + ignored: Some(config.ignored().clone()), + verbose, + ..Default::default() + }, + ) + .wrap_err_with(|| { + let suite = suite.display(); + format!("could not read the test {suite}") + })?; + + match result { + test262::ReadResult::Test(harness, test) => { + if test.edition <= edition { + if verbose != 0 { + println!("Test loaded, starting..."); + } + test.run(&harness, verbose, optimizer_options, console); + } else { + println!( + "Minimum spec edition of test is bigger than the specified edition. Skipping." + ); + } - if test.edition <= edition { + println!(); + Ok(()) + } + test262::ReadResult::TestSuite(harness, suite) => { if verbose != 0 { - println!("Test loaded, starting..."); + println!("Test suite loaded, starting tests..."); } - test.run(&harness, verbose, optimizer_options, console); - } else { - println!( - "Minimum spec edition of test is bigger than the specified edition. Skipping." + let results = suite.run( + &harness, + verbose, + parallel, + edition, + optimizer_options, + console, ); - } - - println!(); - } else { - let suite = - read_suite(&test262_path.join(suite), config.ignored(), false).wrap_err_with(|| { - let suite = suite.display(); - format!("could not read the suite {suite}") - })?; - if verbose != 0 { - println!("Test suite loaded, starting tests..."); - } - let results = suite.run( - &harness, - verbose, - parallel, - edition, - optimizer_options, - console, - ); - - if versioned { - let mut table = comfy_table::Table::new(); - table.load_preset(comfy_table::presets::UTF8_HORIZONTAL_ONLY); - table.set_header(vec![ - "Edition", "Total", "Passed", "Ignored", "Failed", "Panics", "%", - ]); - for column in table.column_iter_mut().skip(1) { - column.set_cell_alignment(comfy_table::CellAlignment::Right); - } - for (v, stats) in SpecEdition::all_editions() - .filter(|v| *v <= edition) - .map(|v| { - let stats = results.versioned_stats.get(v).unwrap_or(results.stats); - (v, stats) - }) - { + if versioned { + let mut table = comfy_table::Table::new(); + table.load_preset(comfy_table::presets::UTF8_HORIZONTAL_ONLY); + table.set_header(vec![ + "Edition", "Total", "Passed", "Ignored", "Failed", "Panics", "%", + ]); + for column in table.column_iter_mut().skip(1) { + column.set_cell_alignment(comfy_table::CellAlignment::Right); + } + for (v, stats) in SpecEdition::all_editions() + .filter(|v| *v <= edition) + .map(|v| { + let stats = results.versioned_stats.get(v).unwrap_or(results.stats); + (v, stats) + }) + { + let Statistics { + total, + passed, + ignored, + panic, + } = stats; + let failed = total - passed - ignored; + let conformance = (passed as f64 / total as f64) * 100.0; + let conformance = format!("{conformance:.2}"); + table.add_row(vec![ + v.to_string(), + total.to_string(), + passed.to_string(), + ignored.to_string(), + failed.to_string(), + panic.to_string(), + conformance, + ]); + } + println!("\n\nResults\n"); + println!("{table}"); + } else { let Statistics { total, passed, ignored, panic, - } = stats; - let failed = total - passed - ignored; - let conformance = (passed as f64 / total as f64) * 100.0; - let conformance = format!("{conformance:.2}"); - table.add_row(vec![ - v.to_string(), - total.to_string(), - passed.to_string(), - ignored.to_string(), - failed.to_string(), - panic.to_string(), - conformance, - ]); + } = results.stats; + println!("\n\nResults ({edition}):"); + println!("Total tests: {total}"); + println!("Passed tests: {}", passed.to_string().green()); + println!("Ignored tests: {}", ignored.to_string().yellow()); + println!( + "Failed tests: {} ({})", + (total - passed - ignored).to_string().red(), + format!("{panic} panics").red() + ); + println!( + "Conformance: {:.2}%", + (passed as f64 / total as f64) * 100.0 + ); + + if let Some(output) = output { + write_json(results, output, verbose, test262_path) + .wrap_err("could not write the results to the output JSON file")?; + } } - println!("\n\nResults\n"); - println!("{table}"); - } else { - let Statistics { - total, - passed, - ignored, - panic, - } = results.stats; - println!("\n\nResults ({edition}):"); - println!("Total tests: {total}"); - println!("Passed tests: {}", passed.to_string().green()); - println!("Ignored tests: {}", ignored.to_string().yellow()); - println!( - "Failed tests: {} ({})", - (total - passed - ignored).to_string().red(), - format!("{panic} panics").red() - ); - println!( - "Conformance: {:.2}%", - (passed as f64 / total as f64) * 100.0 - ); - } - if let Some(output) = output { - write_json(results, output, verbose, test262_path) - .wrap_err("could not write the results to the output JSON file")?; + Ok(()) } } - - Ok(()) -} - -/// All the harness include files. -#[derive(Debug, Clone)] -struct Harness { - assert: HarnessFile, - sta: HarnessFile, - doneprint_handle: HarnessFile, - includes: FxHashMap, HarnessFile>, -} - -#[derive(Debug, Clone)] -struct HarnessFile { - content: Box, - path: Box, -} - -/// Represents a test suite. -#[derive(Debug, Clone)] -struct TestSuite { - name: Box, - path: Box, - suites: Box<[TestSuite]>, - tests: Box<[Test]>, } /// Represents a tests statistic @@ -694,6 +484,7 @@ impl VersionedStats { SpecEdition::ES14 => self.es14, SpecEdition::ES15 => self.es15, SpecEdition::ESNext => return None, + _ => return None, }; Some(stats) } @@ -714,6 +505,7 @@ impl VersionedStats { SpecEdition::ES14 => &mut self.es14, SpecEdition::ES15 => &mut self.es15, SpecEdition::ESNext => return None, + _ => return None, }; Some(stats) } @@ -801,212 +593,35 @@ enum TestOutcomeResult { Panic, } -/// Represents a test. -#[derive(Debug, Clone)] -#[allow(dead_code)] -struct Test { - name: Box, - path: Box, - description: Box, - esid: Option>, - edition: SpecEdition, - flags: TestFlags, - information: Box, - expected_outcome: Outcome, - features: FxHashSet>, - includes: FxHashSet>, - locale: Locale, - ignored: bool, -} - -impl Test { - /// Creates a new test. - fn new(name: N, path: C, metadata: MetaData) -> Result - where - N: Into>, - C: Into>, - { - let edition = SpecEdition::from_test_metadata(&metadata) - .map_err(|feats| eyre!("test metadata contained unknown features: {feats:?}"))?; - - Ok(Self { - edition, - name: name.into(), - description: metadata.description, - esid: metadata.esid, - flags: metadata.flags.into(), - information: metadata.info, - features: metadata.features.into_vec().into_iter().collect(), - expected_outcome: Outcome::from(metadata.negative), - includes: metadata.includes.into_vec().into_iter().collect(), - locale: metadata.locale, - path: path.into(), - ignored: false, - }) - } - - /// Sets the test as ignored. - #[inline] - fn set_ignored(&mut self) { - self.ignored = true; - } - - /// Checks if this is a module test. - #[inline] - const fn is_module(&self) -> bool { - self.flags.contains(TestFlags::MODULE) - } -} - -/// An outcome for a test. -#[derive(Debug, Clone)] -enum Outcome { - Positive, - Negative { phase: Phase, error_type: ErrorType }, -} - -impl Default for Outcome { - fn default() -> Self { - Self::Positive - } -} - -impl From> for Outcome { - fn from(neg: Option) -> Self { - neg.map(|neg| Self::Negative { - phase: neg.phase, - error_type: neg.error_type, - }) - .unwrap_or_default() - } -} - -bitflags! { - #[derive(Debug, Clone, Copy)] - struct TestFlags: u16 { - const STRICT = 0b0_0000_0001; - const NO_STRICT = 0b0_0000_0010; - const MODULE = 0b0_0000_0100; - const RAW = 0b0_0000_1000; - const ASYNC = 0b0_0001_0000; - const GENERATED = 0b0_0010_0000; - const CAN_BLOCK_IS_FALSE = 0b0_0100_0000; - const CAN_BLOCK_IS_TRUE = 0b0_1000_0000; - const NON_DETERMINISTIC = 0b1_0000_0000; - } -} - -impl Default for TestFlags { - fn default() -> Self { - Self::STRICT | Self::NO_STRICT - } -} +#[cfg(test)] +mod tests { + use assert_cmd::Command; -impl From for TestFlags { - fn from(flag: TestFlag) -> Self { - match flag { - TestFlag::OnlyStrict => Self::STRICT, - TestFlag::NoStrict => Self::NO_STRICT, - TestFlag::Module => Self::MODULE, - TestFlag::Raw => Self::RAW, - TestFlag::Async => Self::ASYNC, - TestFlag::Generated => Self::GENERATED, - TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, - TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, - TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, - } + fn cmd() -> Command { + let mut cmd: Command = Command::cargo_bin("boa_tester").unwrap(); + cmd.current_dir("../../"); + cmd } -} - -impl From for TestFlags -where - T: AsRef<[TestFlag]>, -{ - fn from(flags: T) -> Self { - let flags = flags.as_ref(); - if flags.is_empty() { - Self::default() - } else { - let mut result = Self::empty(); - for flag in flags { - result |= Self::from(*flag); - } - if !result.intersects(Self::default()) { - result |= Self::default(); - } + #[test] + #[ignore = "manual"] + fn test_help() { + let output = cmd().output().unwrap(); + let err = String::from_utf8(output.stderr).unwrap(); + assert!(err.contains("boa_tester ")); - result - } + cmd().arg("run").arg("--help").assert().success(); + cmd().arg("compare").arg("--help").assert().success(); } -} - -impl<'de> Deserialize<'de> for TestFlags { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct FlagsVisitor; - - impl<'de> Visitor<'de> for FlagsVisitor { - type Value = TestFlags; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a sequence of flags") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut flags = TestFlags::empty(); - while let Some(elem) = seq.next_element::()? { - flags |= elem.into(); - } - Ok(flags) - } - } - - struct RawFlagsVisitor; - - impl Visitor<'_> for RawFlagsVisitor { - type Value = TestFlags; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a flags number") - } - - fn visit_u16(self, v: u16) -> Result - where - E: serde::de::Error, - { - TestFlags::from_bits(v).ok_or_else(|| { - E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") - }) - } - } - if deserializer.is_human_readable() { - deserializer.deserialize_seq(FlagsVisitor) - } else { - deserializer.deserialize_u16(RawFlagsVisitor) - } + #[test] + #[ignore = "manual"] + fn test() { + cmd() + .arg("run") + .arg("--edition=es5") + .arg("-O") + .assert() + .success(); } } - -/// Phase for an error. -#[derive(Debug, Clone, Copy, Deserialize)] -#[serde(rename_all = "lowercase")] -enum Phase { - Parse, - Resolution, - Runtime, -} - -/// Locale information structure. -#[derive(Debug, Default, Clone, Deserialize)] -#[serde(transparent)] -#[allow(dead_code)] -struct Locale { - locale: Box<[Box]>, -} diff --git a/tests/tester/src/read.rs b/tests/tester/src/read.rs deleted file mode 100644 index 68a9856cf51..00000000000 --- a/tests/tester/src/read.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Module to read the list of test suites from disk. - -use std::{ - ffi::OsStr, - fs, - path::{Path, PathBuf}, -}; - -use color_eyre::{ - eyre::{OptionExt, WrapErr}, - Result, -}; -use rustc_hash::FxHashMap; -use serde::Deserialize; - -use crate::{HarnessFile, Ignored}; - -use super::{Harness, Locale, Phase, Test, TestSuite}; - -/// Representation of the YAML metadata in Test262 tests. -#[derive(Debug, Clone, Deserialize)] -pub(super) struct MetaData { - pub(super) description: Box, - pub(super) esid: Option>, - #[allow(dead_code)] - pub(super) es5id: Option>, - pub(super) es6id: Option>, - #[serde(default)] - pub(super) info: Box, - #[serde(default)] - pub(super) features: Box<[Box]>, - #[serde(default)] - pub(super) includes: Box<[Box]>, - #[serde(default)] - pub(super) flags: Box<[TestFlag]>, - #[serde(default)] - pub(super) negative: Option, - #[serde(default)] - pub(super) locale: Locale, -} - -/// Negative test information structure. -#[derive(Debug, Clone, Deserialize)] -pub(super) struct Negative { - pub(super) phase: Phase, - #[serde(rename = "type")] - pub(super) error_type: ErrorType, -} - -/// All possible error types -#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)] -#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants -pub(super) enum ErrorType { - Test262Error, - SyntaxError, - ReferenceError, - RangeError, - TypeError, -} - -impl ErrorType { - pub(super) const fn as_str(self) -> &'static str { - match self { - Self::Test262Error => "Test262Error", - Self::SyntaxError => "SyntaxError", - Self::ReferenceError => "ReferenceError", - Self::RangeError => "RangeError", - Self::TypeError => "TypeError", - } - } -} - -/// Individual test flag. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(super) enum TestFlag { - OnlyStrict, - NoStrict, - Module, - Raw, - Async, - Generated, - #[serde(rename = "CanBlockIsFalse")] - CanBlockIsFalse, - #[serde(rename = "CanBlockIsTrue")] - CanBlockIsTrue, - #[serde(rename = "non-deterministic")] - NonDeterministic, -} - -/// Reads the Test262 defined bindings. -pub(super) fn read_harness(test262_path: &Path) -> Result { - fn read_harness_file(path: PathBuf) -> Result { - let content = fs::read_to_string(path.as_path()) - .wrap_err_with(|| format!("error reading the harness file `{}`", path.display()))?; - - Ok(HarnessFile { - content: content.into_boxed_str(), - path: path.into_boxed_path(), - }) - } - let mut includes = FxHashMap::default(); - - for entry in fs::read_dir(test262_path.join("harness")) - .wrap_err("error reading the harness directory")? - { - let entry = entry?; - let file_name = entry.file_name(); - let file_name = file_name.to_string_lossy(); - - if file_name == "assert.js" || file_name == "sta.js" || file_name == "doneprintHandle.js" { - continue; - } - - includes.insert( - file_name.into_owned().into_boxed_str(), - read_harness_file(entry.path())?, - ); - } - let assert = read_harness_file(test262_path.join("harness/assert.js"))?; - let sta = read_harness_file(test262_path.join("harness/sta.js"))?; - let doneprint_handle = read_harness_file(test262_path.join("harness/doneprintHandle.js"))?; - - Ok(Harness { - assert, - sta, - doneprint_handle, - includes, - }) -} - -/// Reads a test suite in the given path. -pub(super) fn read_suite( - path: &Path, - ignored: &Ignored, - mut ignore_suite: bool, -) -> Result { - let name = path - .file_name() - .and_then(OsStr::to_str) - .ok_or_eyre("invalid path for test suite")?; - - ignore_suite |= ignored.contains_test(name); - - let mut suites = Vec::new(); - let mut tests = Vec::new(); - - // TODO: iterate in parallel - for entry in path.read_dir().wrap_err("could not retrieve entry")? { - let entry = entry?; - let filetype = entry.file_type().wrap_err("could not retrieve file type")?; - - if filetype.is_dir() { - suites.push( - read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| { - let path = entry.path(); - let suite = path.display(); - format!("error reading sub-suite {suite}") - })?, - ); - continue; - } - - let path = entry.path(); - - if path.extension() != Some(OsStr::new("js")) { - // Ignore files that aren't executable. - continue; - } - - if path - .file_stem() - .is_some_and(|stem| stem.as_encoded_bytes().ends_with(b"FIXTURE")) - { - // Ignore files that are fixtures. - continue; - } - - let mut test = read_test(&path).wrap_err_with(|| { - let path = entry.path(); - let suite = path.display(); - format!("error reading test {suite}") - })?; - - if ignore_suite - || ignored.contains_any_flag(test.flags) - || ignored.contains_test(&test.name) - || test - .features - .iter() - .any(|feat| ignored.contains_feature(feat)) - { - test.set_ignored(); - } - tests.push(test); - } - - Ok(TestSuite { - name: name.into(), - path: Box::from(path), - suites: suites.into_boxed_slice(), - tests: tests.into_boxed_slice(), - }) -} - -/// Reads information about a given test case. -pub(super) fn read_test(path: &Path) -> Result { - let name = path - .file_stem() - .and_then(OsStr::to_str) - .ok_or_eyre("invalid path for test")?; - - let metadata = read_metadata(path)?; - - Test::new(name, path, metadata) -} - -/// Reads the metadata from the input test code. -fn read_metadata(test: &Path) -> Result { - let code = fs::read_to_string(test)?; - - let (_, metadata) = code - .split_once("/*---") - .ok_or_eyre("invalid test metadata")?; - let (metadata, _) = metadata - .split_once("---*/") - .ok_or_eyre("invalid test metadata")?; - let metadata = metadata.replace('\r', "\n"); - - serde_yaml::from_str(&metadata).map_err(Into::into) -} diff --git a/tools/test262/.gitignore b/tools/test262/.gitignore new file mode 100644 index 00000000000..325b51d3910 --- /dev/null +++ b/tools/test262/.gitignore @@ -0,0 +1,2 @@ +/test262 +/test262_2 diff --git a/tools/test262/Cargo.toml b/tools/test262/Cargo.toml new file mode 100644 index 00000000000..22cbd29ac5c --- /dev/null +++ b/tools/test262/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "test262" +edition.workspace = true +version.workspace = true +rust-version.workspace = true +authors.workspace = true +repository.workspace = true +license.workspace = true +description.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { workspace = true, features = ["derive"] } +phf = { workspace = true, features = ["macros"] } +rustc-hash = { workspace = true, features = ["std"] } +bitflags.workspace = true +color-eyre.workspace = true +serde_yaml = "0.9.34" # TODO: Track https://github.com/saphyr-rs/saphyr. + +[dev-dependencies] +serde_json = "1" + +[lints] +workspace = true diff --git a/tests/tester/src/edition.rs b/tools/test262/src/edition.rs similarity index 80% rename from tests/tester/src/edition.rs rename to tools/test262/src/edition.rs index 110169dd9b9..242ec0366d8 100644 --- a/tests/tester/src/edition.rs +++ b/tools/test262/src/edition.rs @@ -3,12 +3,9 @@ //! This module contains the [`SpecEdition`] struct, which is used in the tester to //! classify all tests per minimum required ECMAScript edition. +use crate::{test_flags::TestFlag, MetaData}; use std::fmt::Display; -use serde_repr::{Deserialize_repr, Serialize_repr}; - -use crate::read::{MetaData, TestFlag}; - /// Minimum edition required by a specific feature in the `test262` repository. static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { // Proposed language features @@ -279,69 +276,104 @@ static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { /// List of ECMAScript editions that can be tested in the `test262` repository. #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Default, - Serialize_repr, - Deserialize_repr, - clap::ValueEnum, + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, )] -#[repr(u8)] -pub(crate) enum SpecEdition { +#[serde(transparent)] +#[repr(transparent)] +pub struct SpecEdition(u8); + +impl Default for SpecEdition { + fn default() -> Self { + SpecEdition::ESNext + } +} + +impl SpecEdition { + /// Deserialize SpecEditon from string (label) for example `es5` + pub fn from_label(value: &str) -> Result { + match &*value.to_uppercase() { + "ES5" => Ok(SpecEdition::ES5), + "ES6" => Ok(SpecEdition::ES6), + "ES7" => Ok(SpecEdition::ES7), + "ES8" => Ok(SpecEdition::ES8), + "ES9" => Ok(SpecEdition::ES9), + "ES10" => Ok(SpecEdition::ES10), + "ES11" => Ok(SpecEdition::ES11), + "ES12" => Ok(SpecEdition::ES12), + "ES13" => Ok(SpecEdition::ES13), + "ES14" => Ok(SpecEdition::ES14), + "ES15" => Ok(SpecEdition::ES15), + "ESNEXT" => Ok(SpecEdition::ESNext), + _ => { + if let Ok(nr) = value.parse::() { + if nr >= 5 && nr <= 15 { + return Ok(SpecEdition(nr)); + } + } + + Err("Invalid SpecEdition label".to_string()) + } + } + } +} + +// clap arg parser +impl From<&str> for SpecEdition { + fn from(value: &str) -> Self { + SpecEdition::from_label(value).unwrap_or_default() + } +} + +impl SpecEdition { /// ECMAScript 5.1 Edition /// /// - ES5 = 5, + pub const ES5: SpecEdition = SpecEdition(5); /// ECMAScript 6th Edition /// /// - ES6, + pub const ES6: SpecEdition = SpecEdition(6); /// ECMAScript 7th Edition /// /// - ES7, + pub const ES7: SpecEdition = SpecEdition(7); /// ECMAScript 8th Edition /// /// - ES8, + pub const ES8: SpecEdition = SpecEdition(8); /// ECMAScript 9th Edition /// /// - ES9, + pub const ES9: SpecEdition = SpecEdition(9); /// ECMAScript 10th Edition /// /// - ES10, + pub const ES10: SpecEdition = SpecEdition(10); /// ECMAScript 11th Edition /// /// - ES11, + pub const ES11: SpecEdition = SpecEdition(11); /// ECMAScript 12th Edition /// /// - ES12, + pub const ES12: SpecEdition = SpecEdition(12); /// ECMAScript 13th Edition /// /// - ES13, + pub const ES13: SpecEdition = SpecEdition(13); /// ECMAScript 14th Edition /// /// - ES14, + pub const ES14: SpecEdition = SpecEdition(14); /// ECMAScript 15th Edition /// /// - ES15, + pub const ES15: SpecEdition = SpecEdition(15); /// The edition being worked on right now. /// /// A draft is currently available [here](https://tc39.es/ecma262). - #[default] - ESNext = 255, + #[allow(non_upper_case_globals)] + pub const ESNext: SpecEdition = SpecEdition(255); } impl Display for SpecEdition { @@ -349,7 +381,7 @@ impl Display for SpecEdition { match *self { Self::ESNext => write!(f, "ECMAScript Next"), Self::ES5 => write!(f, "ECMAScript 5.1"), - v => write!(f, "ECMAScript {}", v as u8), + v => write!(f, "ECMAScript {}", v.0), } } } @@ -383,12 +415,15 @@ impl SpecEdition { if unknowns.is_empty() { Ok(min_edition) } else { - Err(unknowns) + // TODO - Temporally fallback to ESNext for unknown features. + // Pending on feature->edition map: https://github.com/tc39/test262/issues/4161 + println!("Unknown test262 features in test metadata: {unknowns:?}"); + Ok(SpecEdition::ESNext) } } /// Gets an iterator of all currently available editions. - pub(crate) fn all_editions() -> impl Iterator { + pub fn all_editions() -> impl Iterator { [ Self::ES5, Self::ES6, @@ -406,3 +441,35 @@ impl SpecEdition { .into_iter() } } + +#[cfg(test)] +mod test { + use serde_json::json; + + use super::SpecEdition; + + #[test] + fn serialize_spec_edition() { + let spec: SpecEdition = serde_json::from_str("6").expect("SpecEdition from number"); + assert_eq!(spec, SpecEdition::ES6); + + let spec_nr = serde_json::to_value(SpecEdition::ES13).expect("SpecEdition to number"); + assert_eq!(spec_nr, json!(13)) + } + + #[test] + fn deserialize_from_label() { + assert_eq!( + SpecEdition::from_label("es6").unwrap_or_default(), + SpecEdition::ES6 + ); + assert_eq!( + SpecEdition::from_label("ES6").unwrap_or_default(), + SpecEdition::ES6 + ); + assert_eq!( + SpecEdition::from_label("6").unwrap_or_default(), + SpecEdition::ES6 + ); + } +} diff --git a/tools/test262/src/error.rs b/tools/test262/src/error.rs new file mode 100644 index 00000000000..8c7a4613ce6 --- /dev/null +++ b/tools/test262/src/error.rs @@ -0,0 +1,68 @@ +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Errors +#[allow(missing_docs)] +pub enum Error262 { + InvalidRepo262Path { + path: PathBuf + }, + InvalidSuitePath { + path: PathBuf + }, + + InvalidHarnessDirecory { + path: String, + }, + HarnessFileReadError { + path: String, + }, + FailedToGetFileType { + path: String, + }, + // test + InvalidPathToTest, + SubTestReadError { + path: String, + suite: String, + error: Box, + }, + // test suite + InvalidPathToTestSuite, + SubSuiteReadError { + path: String, + suite: String, + error: Box, + }, + // test metadata + MetadataUnknownFeatures(Vec), + MetadateReadError { + path: String, + }, + MetadateParseError { + path: String, + }, +} +impl std::error::Error for Error262 {} + +impl std::fmt::Display for Error262 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +pub(super) trait PathToString { + fn string(&self) -> String; +} + +impl PathToString for PathBuf { + fn string(&self) -> String { + self.display().to_string() + } +} + +impl PathToString for Path { + fn string(&self) -> String { + self.display().to_string() + } +} diff --git a/tools/test262/src/git.rs b/tools/test262/src/git.rs new file mode 100644 index 00000000000..aceeffbf2a9 --- /dev/null +++ b/tools/test262/src/git.rs @@ -0,0 +1,148 @@ +//! Git common + +use color_eyre::{eyre::bail, Result}; +use std::{path::Path, process::Command}; + +/// Returns the commit hash and commit message of the provided branch name. +fn get_last_branch_commit(directory: &str, branch: &str, verbose: u8) -> Result<(String, String)> { + if verbose > 1 { + println!("Getting last commit on '{branch}' branch"); + } + let result = Command::new("git") + .arg("log") + .args(["-n", "1"]) + .arg("--pretty=format:%H %s") + .arg(branch) + .current_dir(directory) + .output()?; + + if !result.status.success() { + bail!( + "{directory} getting commit hash and message failed with return code {:?}", + result.status.code() + ); + } + + let output = std::str::from_utf8(&result.stdout)?.trim(); + + let (hash, message) = output + .split_once(' ') + .expect("git log output to contain hash and message"); + + Ok((hash.into(), message.into())) +} + +fn reset_commit(directory: &str, commit: &str, verbose: u8) -> Result<()> { + if verbose != 0 { + println!("Reset {directory} to commit: {commit}..."); + } + + let result = Command::new("git") + .arg("reset") + .arg("--hard") + .arg(commit) + .current_dir(directory) + .status()?; + + if !result.success() { + bail!( + "{directory} commit {commit} checkout failed with return code: {:?}", + result.code() + ); + } + + Ok(()) +} + + +// Todo remove eyre + +/// Clone repository +pub(super) fn clone( + directory: &str, + repor_url: &str, // "https://github.com/tc39/test262" + baranch: &str, // "origin/main" + commit: Option<&str>, + verbose: u8, +) -> Result<()> { + let update = commit.is_none(); + + if Path::new(directory).is_dir() { + let (current_commit_hash, current_commit_message) = + get_last_branch_commit(directory, "HEAD", verbose)?; + + if let Some(commit) = commit { + if current_commit_hash == commit { + return Ok(()); + } + } + + if verbose != 0 { + println!("Fetching latest {directory} commits..."); + } + let result = Command::new("git") + .arg("fetch") + .current_dir(directory) + .status()?; + + if !result.success() { + bail!( + "{directory} fetching latest failed with return code {:?}", + result.code() + ); + } + + if let Some(commit) = commit { + println!("{directory} switching to commit {commit}..."); + reset_commit(directory, commit, verbose)?; + return Ok(()); + } + + if verbose != 0 { + println!("Checking latest {directory} with current HEAD..."); + } + let (latest_commit_hash, latest_commit_message) = + get_last_branch_commit(directory, baranch, verbose)?; + + if current_commit_hash != latest_commit_hash { + if update { + println!("Updating {directory} repository:"); + } else { + println!("Warning {directory} repository is not in sync, use '--test262-commit latest' to automatically update it:"); + } + + println!(" Current commit: {current_commit_hash} {current_commit_message}"); + println!(" Latest commit: {latest_commit_hash} {latest_commit_message}"); + + if update { + reset_commit(&directory, &latest_commit_hash, verbose)?; + } + } + + return Ok(()); + } + + println!("Cloning {repor_url} into {directory} ..."); + let result = Command::new("git") + .arg("clone") + .arg(repor_url) + .arg(directory) + .status()?; + + if !result.success() { + bail!( + "Cloning {repor_url} repository failed with return code {:?}", + result.code() + ); + } + + if let Some(commit) = commit { + if verbose != 0 { + println!("Reset {repor_url} to commit: {commit}..."); + } + + reset_commit(directory, commit, verbose)?; + } + + Ok(()) +} diff --git a/tools/test262/src/lib.rs b/tools/test262/src/lib.rs new file mode 100644 index 00000000000..a44c7942d56 --- /dev/null +++ b/tools/test262/src/lib.rs @@ -0,0 +1,270 @@ +//! TC39 test262 +mod edition; +mod error; +mod git; +mod read; +mod structs; +mod test_files; +mod test_flags; + +use std::path::PathBuf; + +pub use edition::SpecEdition; +pub use error::Error262; +pub use structs::{ErrorType, Ignored, Outcome, Phase}; +pub use test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; +pub use test_flags::TestFlags; + +/// Repository Url +pub const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; +/// Git clone directory +pub const TEST262_DIRECTORY: &str = "test262"; + +/// Clone TC39 test262 repostiory +pub fn clone_test262(commit: Option<&str>, verbose: u8) -> color_eyre::Result<()> { + const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; + git::clone( + TEST262_DIRECTORY, + TEST262_REPOSITORY, + &"origin/main", + commit, + verbose, + ) +} + +/// Test Read Result +#[derive(Debug)] +pub enum ReadResult { + /// Single Test + Test(Harness, Test), + /// Test Suite + TestSuite(Harness, TestSuite) +} + +#[derive(Debug)] +/// Test Reading Options +pub struct ReadOptions<'a> { + /// Git 262 repository path: + /// Example: "/home/test262" or "./test262" + pub test262_path: PathBuf, + /// Example: "https://github.com/tc39/test262" + pub test262_repository: Option<&'a str>, + /// Example: "origin/main" + pub test262_branch: Option<&'a str>, + /// Example "de3a117f02e26a53f8b7cb41f6b4a0b8473c5db4" + pub test262_commit: Option<&'a str>, + /// Ignored configuration + pub ignored: Option, + /// Verbose + pub verbose: u8, +} + +impl Default for ReadOptions<'_> { + fn default() -> Self { + Self { + test262_path: PathBuf::from(TEST262_DIRECTORY), + test262_branch: Default::default(), + test262_repository: Default::default(), + test262_commit: Default::default(), + ignored: Default::default(), + verbose: 0, + } + } +} + +/// Read Test/TestSuite from tc38/test262 repository +/// Example: +/// suite: test/intl402/constructors-string-and-single-element-array.js +/// options: Default:default() +pub fn read(suite: PathBuf, options: ReadOptions<'_>) -> Result { + let test_262_path = options.test262_path; + let ignore = options.ignored.unwrap_or_default(); + let verbose = options.verbose; + + // Clone test262 repo + if !test_262_path.exists() { + let repo = options.test262_repository.unwrap_or(TEST262_REPOSITORY); + let branch = options.test262_branch.unwrap_or("origin/main"); + let commit = options.test262_commit; + git::clone( + &test_262_path.to_string_lossy(), + repo, + branch, + commit, + verbose, + ).expect("Failed to clone test262") // TODO + } + + // Repo path validation + if !test_262_path.is_dir() { + return Err(Error262::InvalidRepo262Path { + path: test_262_path, + }); + } + let Ok(test_262_path) = test_262_path.canonicalize() else { + return Err(Error262::InvalidRepo262Path { + path: test_262_path, + }); + }; + + // Test/TestSuite path validation + let absolute_or_relative = suite.canonicalize().or(test_262_path.join(&suite).canonicalize()); + let Ok(suite) = absolute_or_relative else { + return Err(Error262::InvalidSuitePath { path: suite }); + }; + if suite.is_absolute() && !suite.starts_with(&test_262_path) { + return Err(Error262::InvalidSuitePath { path: suite }); + } + + + let harness = Harness::read(&test_262_path).expect("Failed to read Harness"); // TOOD; + + if suite.to_string_lossy().ends_with(".js") { + let test = Test::read(&suite)?; + return Ok(ReadResult::Test(harness, test)); + } else if suite.is_dir() { + let test = TestSuite::read(&suite, &ignore, false)?; + return Ok(ReadResult::TestSuite(harness, test)); + } + + Err(Error262::InvalidSuitePath { path: suite }) +} + +#[cfg(test)] +mod tests { + use crate::edition::SpecEdition; + use crate::{Ignored, MetaData, TEST262_DIRECTORY}; + use std::path::{Path, PathBuf}; + + #[test] + #[ignore = "manual"] + fn should_clone_test262() { + super::clone_test262(None, 0).unwrap(); + } + + #[test] + #[ignore = "manual"] + fn should_read_harness() { + let harness = super::Harness::read(Path::new(TEST262_DIRECTORY)).unwrap(); + assert!(harness.assert.path.is_file()); + assert!(harness.sta.path.is_file()); + assert!(harness.doneprint_handle.path.is_file()); + } + + #[test] + #[ignore = "manual"] + fn should_read_test_suite_and_test() { + let path = Path::new(TEST262_DIRECTORY) + .join("test") + .join("language") + .join("import"); + let test_suite = super::TestSuite::read(&path, &Ignored::default(), false).unwrap(); + assert!(!test_suite.name.is_empty()); + assert!(!test_suite.tests.is_empty()); + + let test_path = &test_suite.tests[0].path; + let test = super::Test::read(test_path); + assert!(test.is_ok()); + } + + #[test] + fn should_ignore_unknown_features() { + let metadata = MetaData { + description: String::into_boxed_str("test_example description".to_string()), + esid: None, + es5id: None, + es6id: None, + info: String::into_boxed_str("test_example".to_string()), + features: Box::new([String::into_boxed_str("unknown_feature_abc".to_string())]), + includes: Box::new([]), + flags: Box::new([]), + negative: None, + locale: Default::default(), + }; + assert_eq!( + Ok(SpecEdition::ESNext), + SpecEdition::from_test_metadata(&metadata) + ); + } + + #[test] + fn should_get_minimal_required_edition_from_test_metadata() { + let metadata = MetaData { + description: String::into_boxed_str("test_example description".to_string()), + esid: None, + es5id: None, + es6id: None, + info: String::into_boxed_str("test_example".to_string()), + features: Box::new( + [ + String::into_boxed_str("TypedArray".to_string()), // ES6 + String::into_boxed_str("well-formed-json-stringify".to_string()), + ], // ES10 + ), + includes: Box::new([]), + flags: Box::new([]), + negative: None, + locale: Default::default(), + }; + assert_eq!( + Ok(SpecEdition::ES10), + SpecEdition::from_test_metadata(&metadata) + ); + } + + #[test] + #[ignore = "manual"] + fn should_read_test_with_path_relative_to_repository() { + let suite = PathBuf::from("test/intl402/constructors-string-and-single-element-array.js"); + let result = super::read(suite, Default::default()).expect("Test"); + match result { + crate::ReadResult::Test(_, test) => { + assert_eq!(test.name.as_ref(), "constructors-string-and-single-element-array"); + }, + _ => unreachable!("ReadResult::Test was expected"), + } + } + + #[test] + #[ignore = "manual"] + fn should_read_test_with_path_relative_to_current_directory() { + let suite = PathBuf::from("test262/test/intl402/constructors-string-and-single-element-array.js"); + let result = super::read(suite, Default::default()).expect("Test"); + match result { + crate::ReadResult::Test(_, test) => { + assert_eq!(test.name.as_ref(), "constructors-string-and-single-element-array"); + }, + _ => unreachable!("ReadResult::Test was expected"), + } + } + + #[test] + #[ignore = "manual"] + fn should_read_test_suite() { + let suite = PathBuf::from("test262/test/intl402/"); + let result = super::read(suite, Default::default()).expect("Test"); + match result { + crate::ReadResult::TestSuite(_, test) => { + assert_eq!(test.name.as_ref(), "intl402"); + }, + _ => unreachable!("ReadResult::TestSuite was expected"), + } + } + + #[test] + #[ignore = "manual"] + fn should_read_test_suite_custom_options() { + let suite = PathBuf::from("test262_2/test/intl402/"); + let result = super::read(suite, crate::ReadOptions { + test262_path: PathBuf::from("./test262_2"), + ..Default::default() + }).expect("Test"); + + match result { + crate::ReadResult::TestSuite(_, test) => { + assert_eq!(test.name.as_ref(), "intl402"); + }, + _ => unreachable!("ReadResult::TestSuite was expected"), + } + } +} diff --git a/tools/test262/src/read.rs b/tools/test262/src/read.rs new file mode 100644 index 00000000000..feb7086c67d --- /dev/null +++ b/tools/test262/src/read.rs @@ -0,0 +1,187 @@ +//! Module to read the list of test suites from disk. +use crate::{error::PathToString, Error262}; + +use super::test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; + +use rustc_hash::FxHashMap; +use std::{ + ffi::OsStr, + fs, + path::{Path, PathBuf}, +}; + +impl Harness { + /// Reads the Test262 defined bindings. + pub fn read(test262_path: &Path) -> Result { + fn read_harness_file(path: PathBuf) -> Result { + let content = + fs::read_to_string(&path).map_err(|_| Error262::HarnessFileReadError { + path: path.string(), + })?; + + Ok(HarnessFile { + content: content.into_boxed_str(), + path: path.into_boxed_path(), + }) + } + let mut includes = FxHashMap::default(); + + let harness_dir = test262_path.join("harness"); + for entry in fs::read_dir(&harness_dir).map_err(|_| Error262::InvalidHarnessDirecory { + path: harness_dir.string(), + })? { + let entry = entry.map_err(|_| Error262::InvalidHarnessDirecory { + path: harness_dir.string(), + })?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + + if file_name == "assert.js" + || file_name == "sta.js" + || file_name == "doneprintHandle.js" + { + continue; + } + + includes.insert( + file_name.into_owned().into_boxed_str(), + read_harness_file(entry.path())?, + ); + } + let assert = read_harness_file(test262_path.join("harness/assert.js"))?; + let sta = read_harness_file(test262_path.join("harness/sta.js"))?; + let doneprint_handle = read_harness_file(test262_path.join("harness/doneprintHandle.js"))?; + + Ok(Harness { + assert, + sta, + doneprint_handle, + includes, + }) + } +} + +impl TestSuite { + /// Reads a test suite in the given path. + pub fn read( + path: &Path, + ignored: &crate::structs::Ignored, + mut ignore_suite: bool, + ) -> Result { + let name = path + .file_name() + .and_then(OsStr::to_str) + .ok_or(Error262::InvalidPathToTestSuite)?; + + ignore_suite |= ignored.contains_test(name); + + let mut suites = Vec::new(); + let mut tests = Vec::new(); + + // TODO: iterate in parallel + for entry in path + .read_dir() + .map_err(|_| Error262::InvalidPathToTestSuite)? + { + let entry = entry.map_err(|_| Error262::InvalidPathToTestSuite)?; + let filetype = entry + .file_type() + .map_err(|_| Error262::FailedToGetFileType { + path: entry.path().string(), + })?; + + if filetype.is_dir() { + suites.push( + TestSuite::read(entry.path().as_path(), ignored, ignore_suite).map_err(|e| { + Error262::SubSuiteReadError { + path: entry.path().string(), + suite: path.string(), + error: Box::new(e), + } + })?, + ); + continue; + } + + let path = entry.path(); + + if path.extension() != Some(OsStr::new("js")) { + // Ignore files that aren't executable. + continue; + } + + if path + .file_stem() + .is_some_and(|stem| stem.as_encoded_bytes().ends_with(b"FIXTURE")) + { + // Ignore files that are fixtures. + continue; + } + + let mut test = Test::read(&path).map_err(|e| Error262::SubTestReadError { + path: entry.path().string(), + suite: path.string(), + error: Box::new(e), + })?; + + if ignore_suite + || ignored.contains_any_flag(test.flags) + || ignored.contains_test(&test.name) + || test + .features + .iter() + .any(|feat| ignored.contains_feature(feat)) + { + test.set_ignored(); + } + tests.push(test); + } + + Ok(TestSuite { + name: name.into(), + path: Box::from(path), + suites: suites.into_boxed_slice(), + tests: tests.into_boxed_slice(), + }) + } +} + +impl Test { + /// Reads information about a given test case. + pub fn read(path: &Path) -> Result { + let name = path + .file_stem() + .and_then(OsStr::to_str) + .ok_or(Error262::InvalidPathToTest)?; + + let metadata = MetaData::read(path)?; + + Test::new(name, path, metadata) + } +} + +impl MetaData { + /// Reads the metadata from the input test code. + pub fn read(test: &Path) -> Result { + let code = fs::read_to_string(test).map_err(|_| Error262::MetadateReadError { + path: test.string(), + })?; + + let (_, metadata) = + code.split_once("/*---") + .ok_or_else(|| Error262::MetadateParseError { + path: test.string(), + })?; + let (metadata, _) = + metadata + .split_once("---*/") + .ok_or_else(|| Error262::MetadateParseError { + path: test.string(), + })?; + let metadata = metadata.replace('\r', "\n"); + + serde_yaml::from_str(&metadata).map_err(|_| Error262::MetadateParseError { + path: test.string(), + }) + } +} diff --git a/tools/test262/src/structs.rs b/tools/test262/src/structs.rs new file mode 100644 index 00000000000..4ebcfac4553 --- /dev/null +++ b/tools/test262/src/structs.rs @@ -0,0 +1,119 @@ +#![allow(missing_docs)] +use rustc_hash::FxHashSet; +use serde::Deserialize; +use crate::test_flags::TestFlags; + +/// All possible error types +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)] +#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants +pub enum ErrorType { + Test262Error, + SyntaxError, + ReferenceError, + RangeError, + TypeError, +} + +impl ErrorType { + /// str representation + pub const fn as_str(self) -> &'static str { + match self { + Self::Test262Error => "Test262Error", + Self::SyntaxError => "SyntaxError", + Self::ReferenceError => "ReferenceError", + Self::RangeError => "RangeError", + Self::TypeError => "TypeError", + } + } +} + +/// Phase for an error. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Phase { + Parse, + Resolution, + Runtime, +} + +/// Negative test information structure. +#[derive(Debug, Clone, Copy, Deserialize)] +pub struct Negative { + pub(super) phase: Phase, + #[serde(rename = "type")] + pub(super) error_type: ErrorType, +} + +/// An outcome for a test. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy)] +pub enum Outcome { + Positive, + Negative { phase: Phase, error_type: ErrorType }, +} + +impl Default for Outcome { + fn default() -> Self { + Self::Positive + } +} + +impl From> for Outcome { + fn from(neg: Option) -> Self { + neg.map(|neg| Self::Negative { + phase: neg.phase, + error_type: neg.error_type, + }) + .unwrap_or_default() + } +} + +/// Structure to allow defining ignored tests, features and files that should +/// be ignored even when reading. +#[derive(Debug, Clone, Deserialize)] +pub struct Ignored { + #[serde(default)] + tests: FxHashSet>, + #[serde(default)] + features: FxHashSet>, + #[serde(default = "TestFlags::empty")] + flags: TestFlags, +} + +impl Ignored { + /// Checks if the ignore list contains the given test name in the list of + /// tests to ignore. + pub(crate) fn contains_test(&self, test: &str) -> bool { + self.tests.contains(test) + } + + /// Checks if the ignore list contains the given feature name in the list + /// of features to ignore. + pub(crate) fn contains_feature(&self, feature: &str) -> bool { + if self.features.contains(feature) { + return true; + } + // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. + // This ensures those are also ignored. + feature + .split('.') + .next() + .is_some_and(|feat| self.features.contains(feat)) + } + + pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool { + flags.intersects(self.flags) + } +} + +impl Default for Ignored { + fn default() -> Self { + Self { + tests: FxHashSet::default(), + features: FxHashSet::default(), + flags: TestFlags::empty(), + } + } +} diff --git a/tools/test262/src/test_files.rs b/tools/test262/src/test_files.rs new file mode 100644 index 00000000000..86b0ba2bc8f --- /dev/null +++ b/tools/test262/src/test_files.rs @@ -0,0 +1,124 @@ +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::Deserialize; +use std::path::Path; + +use super::structs::*; +use crate::{SpecEdition, TestFlags}; + +/// All the harness include files. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Harness { + pub assert: HarnessFile, + pub sta: HarnessFile, + pub doneprint_handle: HarnessFile, + pub includes: FxHashMap, HarnessFile>, +} + +/// Represents harness file. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct HarnessFile { + pub content: Box, + pub path: Box, +} + +/// Represents a test. +#[allow(dead_code)] +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Test { + pub name: Box, + pub path: Box, + pub description: Box, + pub esid: Option>, + pub edition: SpecEdition, + pub flags: TestFlags, + pub information: Box, + pub expected_outcome: Outcome, + pub features: FxHashSet>, + pub includes: FxHashSet>, + pub locale: Locale, + pub ignored: bool, +} + +impl Test { + /// Creates a new test. + pub fn new(name: N, path: C, metadata: MetaData) -> Result + where + N: Into>, + C: Into>, + { + let edition = SpecEdition::from_test_metadata(&metadata).map_err(|feats| { + crate::Error262::MetadataUnknownFeatures(feats.into_iter().map(String::from).collect()) + })?; + + Ok(Self { + edition, + name: name.into(), + description: metadata.description, + esid: metadata.esid, + flags: metadata.flags.into(), + information: metadata.info, + features: metadata.features.into_vec().into_iter().collect(), + expected_outcome: Outcome::from(metadata.negative), + includes: metadata.includes.into_vec().into_iter().collect(), + locale: metadata.locale, + path: path.into(), + ignored: false, + }) + } + + /// Sets the test as ignored. + #[inline] + pub fn set_ignored(&mut self) { + self.ignored = true; + } + + /// Checks if this is a module test. + #[inline] + pub const fn is_module(&self) -> bool { + self.flags.contains(TestFlags::MODULE) + } +} + +/// Represents a test suite. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct TestSuite { + pub name: Box, + pub path: Box, + pub suites: Box<[TestSuite]>, + pub tests: Box<[Test]>, +} + +/// Representation of the YAML metadata in Test262 tests. +#[allow(missing_docs)] +#[derive(Debug, Clone, Deserialize)] +pub struct MetaData { + pub description: Box, + pub esid: Option>, + #[allow(dead_code)] + pub es5id: Option>, + pub es6id: Option>, + #[serde(default)] + pub info: Box, + #[serde(default)] + pub features: Box<[Box]>, + #[serde(default)] + pub includes: Box<[Box]>, + #[serde(default)] + pub flags: Box<[crate::test_flags::TestFlag]>, + #[serde(default)] + pub negative: Option, + #[serde(default)] + pub locale: Locale, +} + +/// Locale information structure. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(transparent)] +#[allow(dead_code)] +pub struct Locale { + locale: Box<[Box]>, +} diff --git a/tools/test262/src/test_flags.rs b/tools/test262/src/test_flags.rs new file mode 100644 index 00000000000..53f38af4651 --- /dev/null +++ b/tools/test262/src/test_flags.rs @@ -0,0 +1,138 @@ +#![allow(missing_docs)] +use bitflags::bitflags; +use serde::{ + de::{Unexpected, Visitor}, + Deserialize, Deserializer, +}; + + +/// Individual test flag. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TestFlag { + OnlyStrict, + NoStrict, + Module, + Raw, + Async, + Generated, + #[serde(rename = "CanBlockIsFalse")] + CanBlockIsFalse, + #[serde(rename = "CanBlockIsTrue")] + CanBlockIsTrue, + #[serde(rename = "non-deterministic")] + NonDeterministic, +} + +bitflags! { + #[derive(Debug, Clone, Copy)] + pub struct TestFlags: u16 { + const STRICT = 0b0_0000_0001; + const NO_STRICT = 0b0_0000_0010; + const MODULE = 0b0_0000_0100; + const RAW = 0b0_0000_1000; + const ASYNC = 0b0_0001_0000; + const GENERATED = 0b0_0010_0000; + const CAN_BLOCK_IS_FALSE = 0b0_0100_0000; + const CAN_BLOCK_IS_TRUE = 0b0_1000_0000; + const NON_DETERMINISTIC = 0b1_0000_0000; + } +} + +impl Default for TestFlags { + fn default() -> Self { + Self::STRICT | Self::NO_STRICT + } +} + +impl From for TestFlags { + fn from(flag: TestFlag) -> Self { + match flag { + TestFlag::OnlyStrict => Self::STRICT, + TestFlag::NoStrict => Self::NO_STRICT, + TestFlag::Module => Self::MODULE, + TestFlag::Raw => Self::RAW, + TestFlag::Async => Self::ASYNC, + TestFlag::Generated => Self::GENERATED, + TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, + TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, + TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, + } + } +} + +impl From for TestFlags +where + T: AsRef<[TestFlag]>, +{ + fn from(flags: T) -> Self { + let flags = flags.as_ref(); + if flags.is_empty() { + Self::default() + } else { + let mut result = Self::empty(); + for flag in flags { + result |= Self::from(*flag); + } + + if !result.intersects(Self::default()) { + result |= Self::default(); + } + + result + } + } +} + +impl<'de> Deserialize<'de> for TestFlags { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct FlagsVisitor; + + impl<'de> Visitor<'de> for FlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a sequence of flags") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut flags = TestFlags::empty(); + while let Some(elem) = seq.next_element::()? { + flags |= elem.into(); + } + Ok(flags) + } + } + + struct RawFlagsVisitor; + + impl Visitor<'_> for RawFlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a flags number") + } + + fn visit_u16(self, v: u16) -> Result + where + E: serde::de::Error, + { + TestFlags::from_bits(v).ok_or_else(|| { + E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") + }) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_seq(FlagsVisitor) + } else { + deserializer.deserialize_u16(RawFlagsVisitor) + } + } +}