feat: add dependency

javascript-engine/external/boa/boa_tester/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "boa_tester"
+description = "ECMA-262 tests runner for the Boa JavaScript engine."
+keywords = ["javascript", "ECMASCript", "compiler", "test262", "tester"]
+categories = ["command-line-utilites"]
+publish = false
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+boa_engine.workspace = true 
+boa_gc.workspace = true
+boa_parser.workspace = true
+clap = { version = "4.1.1", features = ["derive"] }
+serde = { version = "1.0.152", features = ["derive"] }
+serde_yaml = "0.9.17"
+serde_json = "1.0.91"
+bitflags = "1.3.2"
+regex = "1.7.1"
+once_cell = "1.17.0"
+colored = "2.0.0"
+fxhash = "0.2.1"
+rayon = "1.6.1"
+toml = "0.5.11"
+color-eyre = "0.6.2"
+
+[features]
+default = ["intl"]
+intl = ["boa_engine/intl"]

javascript-engine/external/boa/boa_tester/src/exec/js262.rs

@@ -0,0 +1,107 @@
+use boa_engine::{
+    builtins::JsArgs,
+    object::{JsObject, ObjectInitializer},
+    property::Attribute,
+    Context, JsNativeError, JsResult, JsValue,
+};
+
+/// Initializes the object in the context.
+pub(super) fn init(context: &mut Context<'_>) -> JsObject {
+    let global_obj = context.global_object().clone();
+
+    let obj = ObjectInitializer::new(context)
+        .function(create_realm, "createRealm", 0)
+        .function(detach_array_buffer, "detachArrayBuffer", 2)
+        .function(eval_script, "evalScript", 1)
+        .function(gc, "gc", 0)
+        .property("global", global_obj, Attribute::default())
+        // .property("agent", agent, Attribute::default())
+        .build();
+
+    context.register_global_property("$262", obj.clone(), Attribute::empty());
+
+    obj
+}
+
+/// The `$262.createRealm()` function.
+///
+/// Creates a new ECMAScript Realm, defines this API on the new realm's global object, and
+/// returns the `$262` property of the new realm's global object.
+#[allow(clippy::unnecessary_wraps)]
+fn create_realm(_this: &JsValue, _: &[JsValue], _context: &mut Context<'_>) -> JsResult<JsValue> {
+    let mut context = Context::default();
+
+    // add the $262 object.
+    let js_262 = init(&mut context);
+
+    Ok(JsValue::new(js_262))
+}
+
+/// The `$262.detachArrayBuffer()` function.
+///
+/// Implements the `DetachArrayBuffer` abstract operation.
+fn detach_array_buffer(
+    _this: &JsValue,
+    args: &[JsValue],
+    _: &mut Context<'_>,
+) -> JsResult<JsValue> {
+    fn type_err() -> JsNativeError {
+        JsNativeError::typ().with_message("The provided object was not an ArrayBuffer")
+    }
+
+    let array_buffer = args
+        .get(0)
+        .and_then(JsValue::as_object)
+        .ok_or_else(type_err)?;
+    let mut array_buffer = array_buffer.borrow_mut();
+    let array_buffer = array_buffer.as_array_buffer_mut().ok_or_else(type_err)?;
+
+    // 1. Assert: IsSharedArrayBuffer(arrayBuffer) is false. TODO
+    // 2. If key is not present, set key to undefined.
+    let key = args.get_or_undefined(1);
+
+    // 3. If SameValue(arrayBuffer.[[ArrayBufferDetachKey]], key) is false, throw a TypeError exception.
+    if !JsValue::same_value(&array_buffer.array_buffer_detach_key, key) {
+        return Err(JsNativeError::typ()
+            .with_message("Cannot detach array buffer with different key")
+            .into());
+    }
+
+    // 4. Set arrayBuffer.[[ArrayBufferData]] to null.
+    array_buffer.array_buffer_data = None;
+
+    // 5. Set arrayBuffer.[[ArrayBufferByteLength]] to 0.
+    array_buffer.array_buffer_byte_length = 0;
+
+    // 6. Return NormalCompletion(null).
+    Ok(JsValue::null())
+}
+
+/// The `$262.evalScript()` function.
+///
+/// Accepts a string value as its first argument and executes it as an ECMAScript script.
+fn eval_script(_this: &JsValue, args: &[JsValue], context: &mut Context<'_>) -> JsResult<JsValue> {
+    args.get(0).and_then(JsValue::as_string).map_or_else(
+        || Ok(JsValue::undefined()),
+        |source_text| match context.parse(source_text.to_std_string_escaped()) {
+            // TODO: check strict
+            Err(e) => Err(JsNativeError::typ()
+                .with_message(format!("Uncaught Syntax Error: {e}"))
+                .into()),
+            // Calling eval here parses the code a second time.
+            // TODO: We can fix this after we have have defined the public api for the vm executer.
+            Ok(_) => context.eval(source_text.to_std_string_escaped()),
+        },
+    )
+}
+
+/// The `$262.gc()` function.
+///
+/// Wraps the host's garbage collection invocation mechanism, if such a capability exists.
+/// Must throw an exception if no capability exists. This is necessary for testing the
+/// semantics of any feature that relies on garbage collection, e.g. the `WeakRef` API.
+#[allow(clippy::unnecessary_wraps)]
+fn gc(_this: &JsValue, _: &[JsValue], _context: &mut Context<'_>) -> JsResult<JsValue> {
+    boa_gc::force_collect();
+    Ok(JsValue::undefined())
+}

javascript-engine/external/boa/boa_tester/src/exec/mod.rs

@@ -0,0 +1,405 @@
+//! Execution module for the test runner.
+
+mod js262;
+
+use super::{
+    Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult, TestResult, TestSuite,
+};
+use crate::read::ErrorType;
+use boa_engine::{
+    builtins::JsArgs, context::ContextBuilder, job::SimpleJobQueue,
+    native_function::NativeFunction, object::FunctionObjectBuilder, property::Attribute, Context,
+    JsNativeErrorKind, JsValue,
+};
+use boa_parser::Parser;
+use colored::Colorize;
+use rayon::prelude::*;
+use std::{borrow::Cow, cell::RefCell, rc::Rc};
+
+impl TestSuite {
+    /// Runs the test suite.
+    pub(crate) fn run(&self, harness: &Harness, verbose: u8, parallel: bool) -> SuiteResult {
+        if verbose != 0 {
+            println!("Suite {}:", self.name);
+        }
+
+        let suites: Vec<_> = if parallel {
+            self.suites
+                .par_iter()
+                .map(|suite| suite.run(harness, verbose, parallel))
+                .collect()
+        } else {
+            self.suites
+                .iter()
+                .map(|suite| suite.run(harness, verbose, parallel))
+                .collect()
+        };
+
+        let tests: Vec<_> = if parallel {
+            self.tests
+                .par_iter()
+                .flat_map(|test| test.run(harness, verbose))
+                .collect()
+        } else {
+            self.tests
+                .iter()
+                .flat_map(|test| test.run(harness, verbose))
+                .collect()
+        };
+
+        let mut features = Vec::new();
+        for test_iter in self.tests.iter() {
+            for feature_iter in test_iter.features.iter() {
+                features.push(feature_iter.to_string());
+            }
+        }
+
+        if verbose != 0 {
+            println!();
+        }
+
+        // Count passed tests
+        let mut passed = 0;
+        let mut ignored = 0;
+        let mut panic = 0;
+        for test in &tests {
+            match test.result {
+                TestOutcomeResult::Passed => passed += 1,
+                TestOutcomeResult::Ignored => ignored += 1,
+                TestOutcomeResult::Panic => panic += 1,
+                TestOutcomeResult::Failed => {}
+            }
+        }
+
+        // Count total tests
+        let mut total = tests.len();
+        for suite in &suites {
+            total += suite.total;
+            passed += suite.passed;
+            ignored += suite.ignored;
+            panic += suite.panic;
+            features.append(&mut suite.features.clone());
+        }
+
+        if verbose != 0 {
+            println!(
+                "Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \
+                    {}{}), conformance: {:.2}%",
+                self.name,
+                passed.to_string().green(),
+                ignored.to_string().yellow(),
+                (total - passed - ignored).to_string().red(),
+                if panic == 0 {
+                    "0".normal()
+                } else {
+                    panic.to_string().red()
+                },
+                if panic == 0 { "" } else { " ⚠" }.red(),
+                (passed as f64 / total as f64) * 100.0
+            );
+        }
+
+        SuiteResult {
+            name: self.name.clone(),
+            total,
+            passed,
+            ignored,
+            panic,
+            suites,
+            tests,
+            features,
+        }
+    }
+}
+
+impl Test {
+    /// Runs the test.
+    pub(crate) fn run(&self, harness: &Harness, verbose: u8) -> Vec<TestResult> {
+        let mut results = Vec::new();
+        if self.flags.contains(TestFlags::STRICT) && !self.flags.contains(TestFlags::RAW) {
+            results.push(self.run_once(harness, true, verbose));
+        }
+
+        if self.flags.contains(TestFlags::NO_STRICT) || self.flags.contains(TestFlags::RAW) {
+            results.push(self.run_once(harness, false, verbose));
+        }
+
+        results
+    }
+
+    /// Runs the test once, in strict or non-strict mode
+    fn run_once(&self, harness: &Harness, strict: bool, verbose: u8) -> TestResult {
+        if self.ignored {
+            if verbose > 1 {
+                println!(
+                    "`{}`{}: {}",
+                    self.name,
+                    if strict { " (strict mode)" } else { "" },
+                    "Ignored".yellow()
+                );
+            } else {
+                print!("{}", "-".yellow());
+            }
+            return TestResult {
+                name: self.name.clone(),
+                strict,
+                result: TestOutcomeResult::Ignored,
+                result_text: Box::default(),
+            };
+        }
+        if verbose > 1 {
+            println!(
+                "`{}`{}: starting",
+                self.name,
+                if strict { " (strict mode)" } else { "" }
+            );
+        }
+
+        let test_content = if strict {
+            Cow::Owned(format!("\"use strict\";\n{}", self.content))
+        } else {
+            Cow::Borrowed(&*self.content)
+        };
+
+        let result = std::panic::catch_unwind(|| match self.expected_outcome {
+            Outcome::Positive => {
+                let queue = SimpleJobQueue::new();
+                let context = &mut ContextBuilder::new().job_queue(&queue).build();
+                let async_result = AsyncResult::default();
+
+                if let Err(e) = self.set_up_env(harness, context, async_result.clone()) {
+                    return (false, e);
+                }
+
+                // TODO: timeout
+                let value = match context.eval(&*test_content) {
+                    Ok(v) => v,
+                    Err(e) => return (false, format!("Uncaught {e}")),
+                };
+
+                context.run_jobs();
+
+                if let Err(e) = async_result.inner.borrow().as_ref() {
+                    return (false, format!("Uncaught {e}"));
+                }
+
+                (true, value.display().to_string())
+            }
+            Outcome::Negative {
+                phase: Phase::Parse | Phase::Early,
+                error_type,
+            } => {
+                assert_eq!(
+                    error_type,
+                    ErrorType::SyntaxError,
+                    "non-SyntaxError parsing/early error found in {}",
+                    self.name
+                );
+
+                let mut context = Context::default();
+                match context.parse(&*test_content) {
+                    Ok(statement_list) => match context.compile(&statement_list) {
+                        Ok(_) => (false, "StatementList compilation should fail".to_owned()),
+                        Err(e) => (true, format!("Uncaught {e:?}")),
+                    },
+                    Err(e) => (true, format!("Uncaught {e}")),
+                }
+            }
+            Outcome::Negative {
+                phase: Phase::Resolution,
+                error_type: _,
+            } => todo!("check module resolution errors"),
+            Outcome::Negative {
+                phase: Phase::Runtime,
+                error_type,
+            } => {
+                let context = &mut Context::default();
+                if let Err(e) = self.set_up_env(harness, context, AsyncResult::default()) {
+                    return (false, e);
+                }
+                let code = match Parser::new(test_content.as_bytes())
+                    .parse_all(context.interner_mut())
+                    .map_err(Into::into)
+                    .and_then(|stmts| context.compile(&stmts))
+                {
+                    Ok(code) => code,
+                    Err(e) => return (false, format!("Uncaught {e}")),
+                };
+
+                let e = match context.execute(code) {
+                    Ok(res) => return (false, res.display().to_string()),
+                    Err(e) => e,
+                };
+                if let Ok(e) = e.try_native(context) {
+                    match &e.kind {
+                        JsNativeErrorKind::Syntax if error_type == ErrorType::SyntaxError => {}
+                        JsNativeErrorKind::Reference if error_type == ErrorType::ReferenceError => {
+                        }
+                        JsNativeErrorKind::Range if error_type == ErrorType::RangeError => {}
+                        JsNativeErrorKind::Type if error_type == ErrorType::TypeError => {}
+                        _ => return (false, format!("Uncaught {e}")),
+                    }
+                    (true, format!("Uncaught {e}"))
+                } else {
+                    let passed = e
+                        .as_opaque()
+                        .expect("try_native cannot fail if e is not opaque")
+                        .as_object()
+                        .and_then(|o| o.get("constructor", context).ok())
+                        .as_ref()
+                        .and_then(JsValue::as_object)
+                        .and_then(|o| o.get("name", context).ok())
+                        .as_ref()
+                        .and_then(JsValue::as_string)
+                        .map(|s| s == error_type.as_str())
+                        .unwrap_or_default();
+                    (passed, format!("Uncaught {e}"))
+                }
+            }
+        });
+
+        let (result, result_text) = result.map_or_else(
+            |_| {
+                eprintln!("last panic was on test \"{}\"", self.name);
+                (TestOutcomeResult::Panic, String::new())
+            },
+            |(res, text)| {
+                if res {
+                    (TestOutcomeResult::Passed, text)
+                } else {
+                    (TestOutcomeResult::Failed, text)
+                }
+            },
+        );
+
+        if verbose > 1 {
+            println!(
+                "`{}`{}: {}",
+                self.name,
+                if strict { " (strict mode)" } else { "" },
+                if result == TestOutcomeResult::Passed {
+                    "Passed".green()
+                } else if result == TestOutcomeResult::Failed {
+                    "Failed".red()
+                } else {
+                    "⚠ Panic ⚠".red()
+                }
+            );
+        } else {
+            print!(
+                "{}",
+                if result == TestOutcomeResult::Passed {
+                    ".".green()
+                } else {
+                    "F".red()
+                }
+            );
+        }
+
+        if verbose > 2 {
+            println!(
+                "`{}`{}: result text",
+                self.name,
+                if strict { " (strict mode)" } else { "" },
+            );
+            println!("{result_text}");
+            println!();
+        }
+
+        TestResult {
+            name: self.name.clone(),
+            strict,
+            result,
+            result_text: result_text.into_boxed_str(),
+        }
+    }
+
+    /// Sets the environment up to run the test.
+    fn set_up_env(
+        &self,
+        harness: &Harness,
+        context: &mut Context<'_>,
+        async_result: AsyncResult,
+    ) -> Result<(), String> {
+        // Register the print() function.
+        Self::register_print_fn(context, async_result);
+
+        // add the $262 object.
+        let _js262 = js262::init(context);
+
+        if self.flags.contains(TestFlags::RAW) {
+            return Ok(());
+        }
+
+        context
+            .eval(harness.assert.as_ref())
+            .map_err(|e| format!("could not run assert.js:\n{e}"))?;
+        context
+            .eval(harness.sta.as_ref())
+            .map_err(|e| format!("could not run sta.js:\n{e}"))?;
+
+        if self.flags.contains(TestFlags::ASYNC) {
+            context
+                .eval(harness.doneprint_handle.as_ref())
+                .map_err(|e| format!("could not run doneprintHandle.js:\n{e}"))?;
+        }
+
+        for include in self.includes.iter() {
+            context
+                .eval(
+                    harness
+                        .includes
+                        .get(include)
+                        .ok_or_else(|| format!("could not find the {include} include file."))?
+                        .as_ref(),
+                )
+                .map_err(|e| format!("could not run the {include} include file:\nUncaught {e}"))?;
+        }
+
+        Ok(())
+    }
+
+    /// Registers the print function in the context.
+    fn register_print_fn(context: &mut Context<'_>, async_result: AsyncResult) {
+        // We use `FunctionBuilder` to define a closure with additional captures.
+        let js_function = FunctionObjectBuilder::new(
+            context,
+            // SAFETY: `AsyncResult` has only non-traceable captures, making this safe.
+            unsafe {
+                NativeFunction::from_closure(move |_, args, context| {
+                    let message = args
+                        .get_or_undefined(0)
+                        .to_string(context)?
+                        .to_std_string_escaped();
+                    if message != "Test262:AsyncTestComplete" {
+                        *async_result.inner.borrow_mut() = Err(message);
+                    }
+                    Ok(JsValue::undefined())
+                })
+            },
+        )
+        .name("print")
+        .length(1)
+        .build();
+
+        context.register_global_property(
+            "print",
+            js_function,
+            Attribute::WRITABLE | Attribute::NON_ENUMERABLE | Attribute::CONFIGURABLE,
+        );
+    }
+}
+
+/// Object which includes the result of the async operation.
+#[derive(Debug, Clone)]
+struct AsyncResult {
+    inner: Rc<RefCell<Result<(), String>>>,
+}
+
+impl Default for AsyncResult {
+    fn default() -> Self {
+        Self {
+            inner: Rc::new(RefCell::new(Ok(()))),
+        }
+    }
+}

javascript-engine/external/boa/boa_tester/src/main.rs

@@ -0,0 +1,560 @@
+//! Test262 test runner
+//!
+//! This crate will run the full ECMAScript test suite (Test262) and report compliance of the
+//! `boa` engine.
+
+#![doc(
+    html_logo_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg",
+    html_favicon_url = "https://raw.githubusercontent.com/boa-dev/boa/main/assets/logo.svg"
+)]
+#![cfg_attr(not(test), deny(clippy::unwrap_used))]
+#![warn(missing_docs, clippy::dbg_macro)]
+#![deny(
+    // rustc lint groups https://doc.rust-lang.org/rustc/lints/groups.html
+    warnings,
+    future_incompatible,
+    let_underscore,
+    nonstandard_style,
+    rust_2018_compatibility,
+    rust_2018_idioms,
+    rust_2021_compatibility,
+    unused,
+
+    // rustc allowed-by-default lints https://doc.rust-lang.org/rustc/lints/listing/allowed-by-default.html
+    macro_use_extern_crate,
+    meta_variable_misuse,
+    missing_abi,
+    missing_copy_implementations,
+    missing_debug_implementations,
+    non_ascii_idents,
+    noop_method_call,
+    single_use_lifetimes,
+    trivial_casts,
+    trivial_numeric_casts,
+    unreachable_pub,
+    unsafe_op_in_unsafe_fn,
+    unused_crate_dependencies,
+    unused_import_braces,
+    unused_lifetimes,
+    unused_qualifications,
+    unused_tuple_struct_fields,
+    variant_size_differences,
+
+    // rustdoc lints https://doc.rust-lang.org/rustdoc/lints.html
+    rustdoc::broken_intra_doc_links,
+    rustdoc::private_intra_doc_links,
+    rustdoc::missing_crate_level_docs,
+    rustdoc::private_doc_tests,
+    rustdoc::invalid_codeblock_attributes,
+    rustdoc::invalid_rust_codeblocks,
+    rustdoc::bare_urls,
+
+    // clippy categories https://doc.rust-lang.org/clippy/
+    clippy::all,
+    clippy::correctness,
+    clippy::suspicious,
+    clippy::style,
+    clippy::complexity,
+    clippy::perf,
+    clippy::pedantic,
+    clippy::nursery,
+)]
+#![allow(
+    clippy::use_self,
+    clippy::too_many_lines,
+    clippy::redundant_pub_crate,
+    clippy::cast_precision_loss,
+    clippy::cast_possible_wrap
+)]
+
+mod exec;
+mod read;
+mod results;
+
+use self::{
+    read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag},
+    results::{compare_results, write_json},
+};
+use bitflags::bitflags;
+use clap::{ArgAction, Parser, ValueHint};
+use color_eyre::{
+    eyre::{bail, WrapErr},
+    Result,
+};
+use colored::Colorize;
+use fxhash::{FxHashMap, FxHashSet};
+use read::ErrorType;
+use serde::{
+    de::{Unexpected, Visitor},
+    Deserialize, Deserializer, Serialize,
+};
+use std::{
+    fs::{self, File},
+    io::Read,
+    path::{Path, PathBuf},
+};
+
+/// Structure to allow defining ignored tests, features and files that should
+/// be ignored even when reading.
+#[derive(Debug, Deserialize)]
+struct Ignored {
+    #[serde(default)]
+    tests: FxHashSet<Box<str>>,
+    #[serde(default)]
+    features: FxHashSet<Box<str>>,
+    #[serde(default = "TestFlags::empty")]
+    flags: TestFlags,
+}
+
+impl Ignored {
+    /// Checks if the ignore list contains the given test name in the list of
+    /// tests to ignore.
+    pub(crate) fn contains_test(&self, test: &str) -> bool {
+        self.tests.contains(test)
+    }
+
+    /// Checks if the ignore list contains the given feature name in the list
+    /// of features to ignore.
+    pub(crate) fn contains_feature(&self, feature: &str) -> bool {
+        if self.features.contains(feature) {
+            return true;
+        }
+        // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`.
+        // This ensures those are also ignored.
+        feature
+            .split('.')
+            .next()
+            .map(|feat| self.features.contains(feat))
+            .unwrap_or_default()
+    }
+
+    pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool {
+        flags.intersects(self.flags)
+    }
+}
+
+impl Default for Ignored {
+    fn default() -> Self {
+        Self {
+            tests: FxHashSet::default(),
+            features: FxHashSet::default(),
+            flags: TestFlags::empty(),
+        }
+    }
+}
+
+/// Boa test262 tester
+#[derive(Debug, Parser)]
+#[command(author, version, about, name = "Boa test262 tester")]
+enum Cli {
+    /// Run the test suite.
+    Run {
+        /// Whether to show verbose output.
+        #[arg(short, long, action = ArgAction::Count)]
+        verbose: u8,
+
+        /// Path to the Test262 suite.
+        #[arg(long, default_value = "./test262", value_hint = ValueHint::DirPath)]
+        test262_path: PathBuf,
+
+        /// Which specific test or test suite to run. Should be a path relative to the Test262 directory: e.g. "test/language/types/number"
+        #[arg(short, long, default_value = "test", value_hint = ValueHint::AnyPath)]
+        suite: PathBuf,
+
+        /// Optional output folder for the full results information.
+        #[arg(short, long, value_hint = ValueHint::DirPath)]
+        output: Option<PathBuf>,
+
+        /// Execute tests serially
+        #[arg(short, long)]
+        disable_parallelism: bool,
+
+        /// Path to a TOML file with the ignored tests, features, flags and/or files.
+        #[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)]
+        ignored: PathBuf,
+    },
+    /// Compare two test suite results.
+    Compare {
+        /// Base results of the suite.
+        #[arg(value_hint = ValueHint::FilePath)]
+        base: PathBuf,
+
+        /// New results to compare.
+        #[arg(value_hint = ValueHint::FilePath)]
+        new: PathBuf,
+
+        /// Whether to use markdown output
+        #[arg(short, long)]
+        markdown: bool,
+    },
+}
+
+/// Program entry point.
+fn main() -> Result<()> {
+    color_eyre::install()?;
+    match Cli::parse() {
+        Cli::Run {
+            verbose,
+            test262_path,
+            suite,
+            output,
+            disable_parallelism,
+            ignored: ignore,
+        } => run_test_suite(
+            verbose,
+            !disable_parallelism,
+            test262_path.as_path(),
+            suite.as_path(),
+            output.as_deref(),
+            ignore.as_path(),
+        ),
+        Cli::Compare {
+            base,
+            new,
+            markdown,
+        } => compare_results(base.as_path(), new.as_path(), markdown),
+    }
+}
+
+/// Runs the full test suite.
+fn run_test_suite(
+    verbose: u8,
+    parallel: bool,
+    test262_path: &Path,
+    suite: &Path,
+    output: Option<&Path>,
+    ignored: &Path,
+) -> Result<()> {
+    if let Some(path) = output {
+        if path.exists() {
+            if !path.is_dir() {
+                bail!("the output path must be a directory.");
+            }
+        } else {
+            fs::create_dir_all(path).wrap_err("could not create the output directory")?;
+        }
+    }
+
+    let ignored = {
+        let mut input = String::new();
+        let mut f = File::open(ignored).wrap_err("could not open ignored tests file")?;
+        f.read_to_string(&mut input)
+            .wrap_err("could not read ignored tests file")?;
+        toml::from_str(&input).wrap_err("could not decode ignored tests file")?
+    };
+
+    if verbose != 0 {
+        println!("Loading the test suite...");
+    }
+    let harness = read_harness(test262_path).wrap_err("could not read harness")?;
+
+    if suite.to_string_lossy().ends_with(".js") {
+        let test = read_test(&test262_path.join(suite)).wrap_err_with(|| {
+            let suite = suite.display();
+            format!("could not read the test {suite}")
+        })?;
+
+        if verbose != 0 {
+            println!("Test loaded, starting...");
+        }
+        test.run(&harness, verbose);
+
+        println!();
+    } else {
+        let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| {
+            let suite = suite.display();
+            format!("could not read the suite {suite}")
+        })?;
+
+        if verbose != 0 {
+            println!("Test suite loaded, starting tests...");
+        }
+        let results = suite.run(&harness, verbose, parallel);
+
+        println!();
+        println!("Results:");
+        println!("Total tests: {}", results.total);
+        println!("Passed tests: {}", results.passed.to_string().green());
+        println!("Ignored tests: {}", results.ignored.to_string().yellow());
+        println!(
+            "Failed tests: {} (panics: {})",
+            (results.total - results.passed - results.ignored)
+                .to_string()
+                .red(),
+            results.panic.to_string().red()
+        );
+        println!(
+            "Conformance: {:.2}%",
+            (results.passed as f64 / results.total as f64) * 100.0
+        );
+
+        write_json(results, output, verbose)
+            .wrap_err("could not write the results to the output JSON file")?;
+    }
+
+    Ok(())
+}
+
+/// All the harness include files.
+#[derive(Debug, Clone)]
+struct Harness {
+    assert: Box<str>,
+    sta: Box<str>,
+    doneprint_handle: Box<str>,
+    includes: FxHashMap<Box<str>, Box<str>>,
+}
+
+/// Represents a test suite.
+#[derive(Debug, Clone)]
+struct TestSuite {
+    name: Box<str>,
+    suites: Box<[TestSuite]>,
+    tests: Box<[Test]>,
+}
+
+/// Outcome of a test suite.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct SuiteResult {
+    #[serde(rename = "n")]
+    name: Box<str>,
+    #[serde(rename = "c")]
+    total: usize,
+    #[serde(rename = "o")]
+    passed: usize,
+    #[serde(rename = "i")]
+    ignored: usize,
+    #[serde(rename = "p")]
+    panic: usize,
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    #[serde(rename = "s")]
+    suites: Vec<SuiteResult>,
+    #[serde(rename = "t")]
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    tests: Vec<TestResult>,
+    #[serde(rename = "f")]
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    features: Vec<String>,
+}
+
+/// Outcome of a test.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[allow(dead_code)]
+struct TestResult {
+    #[serde(rename = "n")]
+    name: Box<str>,
+    #[serde(rename = "s", default)]
+    strict: bool,
+    #[serde(skip)]
+    result_text: Box<str>,
+    #[serde(rename = "r")]
+    result: TestOutcomeResult,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+enum TestOutcomeResult {
+    #[serde(rename = "O")]
+    Passed,
+    #[serde(rename = "I")]
+    Ignored,
+    #[serde(rename = "F")]
+    Failed,
+    #[serde(rename = "P")]
+    Panic,
+}
+
+/// Represents a test.
+#[derive(Debug, Clone, Default)]
+#[allow(dead_code)]
+struct Test {
+    name: Box<str>,
+    description: Box<str>,
+    esid: Option<Box<str>>,
+    flags: TestFlags,
+    information: Box<str>,
+    features: Box<[Box<str>]>,
+    expected_outcome: Outcome,
+    includes: Box<[Box<str>]>,
+    locale: Locale,
+    content: Box<str>,
+    ignored: bool,
+}
+
+impl Test {
+    /// Creates a new test.
+    fn new<N, C>(name: N, content: C, metadata: MetaData) -> Self
+    where
+        N: Into<Box<str>>,
+        C: Into<Box<str>>,
+    {
+        Self {
+            name: name.into(),
+            description: metadata.description,
+            esid: metadata.esid,
+            flags: metadata.flags.into(),
+            information: metadata.info,
+            features: metadata.features,
+            expected_outcome: Outcome::from(metadata.negative),
+            includes: metadata.includes,
+            locale: metadata.locale,
+            content: content.into(),
+            ignored: false,
+        }
+    }
+
+    fn set_ignored(&mut self) {
+        self.ignored = true;
+    }
+}
+
+/// An outcome for a test.
+#[derive(Debug, Clone)]
+enum Outcome {
+    Positive,
+    Negative { phase: Phase, error_type: ErrorType },
+}
+
+impl Default for Outcome {
+    fn default() -> Self {
+        Self::Positive
+    }
+}
+
+impl From<Option<Negative>> for Outcome {
+    fn from(neg: Option<Negative>) -> Self {
+        neg.map(|neg| Self::Negative {
+            phase: neg.phase,
+            error_type: neg.error_type,
+        })
+        .unwrap_or_default()
+    }
+}
+
+bitflags! {
+    struct TestFlags: u16 {
+        const STRICT = 0b0_0000_0001;
+        const NO_STRICT = 0b0_0000_0010;
+        const MODULE = 0b0_0000_0100;
+        const RAW = 0b0_0000_1000;
+        const ASYNC = 0b0_0001_0000;
+        const GENERATED = 0b0_0010_0000;
+        const CAN_BLOCK_IS_FALSE = 0b0_0100_0000;
+        const CAN_BLOCK_IS_TRUE = 0b0_1000_0000;
+        const NON_DETERMINISTIC = 0b1_0000_0000;
+    }
+}
+
+impl Default for TestFlags {
+    fn default() -> Self {
+        Self::STRICT | Self::NO_STRICT
+    }
+}
+
+impl From<TestFlag> for TestFlags {
+    fn from(flag: TestFlag) -> Self {
+        match flag {
+            TestFlag::OnlyStrict => Self::STRICT,
+            TestFlag::NoStrict => Self::NO_STRICT,
+            TestFlag::Module => Self::MODULE,
+            TestFlag::Raw => Self::RAW,
+            TestFlag::Async => Self::ASYNC,
+            TestFlag::Generated => Self::GENERATED,
+            TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE,
+            TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE,
+            TestFlag::NonDeterministic => Self::NON_DETERMINISTIC,
+        }
+    }
+}
+
+impl<T> From<T> for TestFlags
+where
+    T: AsRef<[TestFlag]>,
+{
+    fn from(flags: T) -> Self {
+        let flags = flags.as_ref();
+        if flags.is_empty() {
+            Self::default()
+        } else {
+            let mut result = Self::empty();
+            for flag in flags {
+                result |= Self::from(*flag);
+            }
+
+            if !result.intersects(Self::default()) {
+                result |= Self::default();
+            }
+
+            result
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for TestFlags {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct FlagsVisitor;
+
+        impl<'de> Visitor<'de> for FlagsVisitor {
+            type Value = TestFlags;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(formatter, "a sequence of flags")
+            }
+
+            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::SeqAccess<'de>,
+            {
+                let mut flags = TestFlags::empty();
+                while let Some(elem) = seq.next_element::<TestFlag>()? {
+                    flags |= elem.into();
+                }
+                Ok(flags)
+            }
+        }
+
+        struct RawFlagsVisitor;
+
+        impl Visitor<'_> for RawFlagsVisitor {
+            type Value = TestFlags;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(formatter, "a flags number")
+            }
+
+            fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                TestFlags::from_bits(v).ok_or_else(|| {
+                    E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number")
+                })
+            }
+        }
+
+        if deserializer.is_human_readable() {
+            deserializer.deserialize_seq(FlagsVisitor)
+        } else {
+            deserializer.deserialize_u16(RawFlagsVisitor)
+        }
+    }
+}
+
+/// Phase for an error.
+#[derive(Debug, Clone, Copy, Deserialize)]
+#[serde(rename_all = "lowercase")]
+enum Phase {
+    Parse,
+    Early,
+    Resolution,
+    Runtime,
+}
+
+/// Locale information structure.
+#[derive(Debug, Default, Clone, Deserialize)]
+#[serde(transparent)]
+#[allow(dead_code)]
+struct Locale {
+    locale: Box<[Box<str>]>,
+}

javascript-engine/external/boa/boa_tester/src/read.rs

@@ -0,0 +1,239 @@
+//! Module to read the list of test suites from disk.
+
+use crate::Ignored;
+
+use super::{Harness, Locale, Phase, Test, TestSuite};
+use color_eyre::{
+    eyre::{eyre, WrapErr},
+    Result,
+};
+use fxhash::FxHashMap;
+use serde::Deserialize;
+use std::{fs, io, path::Path};
+
+/// Representation of the YAML metadata in Test262 tests.
+#[derive(Debug, Clone, Deserialize)]
+#[allow(dead_code)]
+pub(super) struct MetaData {
+    pub(super) description: Box<str>,
+    pub(super) esid: Option<Box<str>>,
+    pub(super) es5id: Option<Box<str>>,
+    pub(super) es6id: Option<Box<str>>,
+    #[serde(default)]
+    pub(super) info: Box<str>,
+    #[serde(default)]
+    pub(super) features: Box<[Box<str>]>,
+    #[serde(default)]
+    pub(super) includes: Box<[Box<str>]>,
+    #[serde(default)]
+    pub(super) flags: Box<[TestFlag]>,
+    #[serde(default)]
+    pub(super) negative: Option<Negative>,
+    #[serde(default)]
+    pub(super) locale: Locale,
+}
+
+/// Negative test information structure.
+#[derive(Debug, Clone, Deserialize)]
+pub(super) struct Negative {
+    pub(super) phase: Phase,
+    #[serde(rename = "type")]
+    pub(super) error_type: ErrorType,
+}
+
+/// All possible error types
+#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)]
+#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants
+pub(super) enum ErrorType {
+    Test262Error,
+    SyntaxError,
+    ReferenceError,
+    RangeError,
+    TypeError,
+}
+
+impl ErrorType {
+    pub(super) const fn as_str(self) -> &'static str {
+        match self {
+            Self::Test262Error => "Test262Error",
+            Self::SyntaxError => "SyntaxError",
+            Self::ReferenceError => "ReferenceError",
+            Self::RangeError => "RangeError",
+            Self::TypeError => "TypeError",
+        }
+    }
+}
+
+/// Individual test flag.
+#[derive(Debug, Clone, Copy, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(super) enum TestFlag {
+    OnlyStrict,
+    NoStrict,
+    Module,
+    Raw,
+    Async,
+    Generated,
+    #[serde(rename = "CanBlockIsFalse")]
+    CanBlockIsFalse,
+    #[serde(rename = "CanBlockIsTrue")]
+    CanBlockIsTrue,
+    #[serde(rename = "non-deterministic")]
+    NonDeterministic,
+}
+
+/// Reads the Test262 defined bindings.
+pub(super) fn read_harness(test262_path: &Path) -> Result<Harness> {
+    let mut includes = FxHashMap::default();
+
+    for entry in fs::read_dir(test262_path.join("harness"))
+        .wrap_err("error reading the harness directory")?
+    {
+        let entry = entry?;
+        let file_name = entry.file_name();
+        let file_name = file_name.to_string_lossy();
+
+        if file_name == "assert.js" || file_name == "sta.js" || file_name == "doneprintHandle.js" {
+            continue;
+        }
+
+        let content = fs::read_to_string(entry.path())
+            .wrap_err_with(|| format!("error reading the harnes/{file_name}"))?;
+
+        includes.insert(
+            file_name.into_owned().into_boxed_str(),
+            content.into_boxed_str(),
+        );
+    }
+    let assert = fs::read_to_string(test262_path.join("harness/assert.js"))
+        .wrap_err("error reading harnes/assert.js")?
+        .into_boxed_str();
+    let sta = fs::read_to_string(test262_path.join("harness/sta.js"))
+        .wrap_err("error reading harnes/sta.js")?
+        .into_boxed_str();
+    let doneprint_handle = fs::read_to_string(test262_path.join("harness/doneprintHandle.js"))
+        .wrap_err("error reading harnes/doneprintHandle.js")?
+        .into_boxed_str();
+
+    Ok(Harness {
+        assert,
+        sta,
+        doneprint_handle,
+        includes,
+    })
+}
+
+/// Reads a test suite in the given path.
+pub(super) fn read_suite(
+    path: &Path,
+    ignored: &Ignored,
+    mut ignore_suite: bool,
+) -> Result<TestSuite> {
+    let name = path
+        .file_name()
+        .ok_or_else(|| eyre!(format!("test suite with no name found: {}", path.display())))?
+        .to_str()
+        .ok_or_else(|| eyre!(format!("non-UTF-8 suite name found: {}", path.display())))?;
+
+    ignore_suite |= ignored.contains_test(name);
+
+    let mut suites = Vec::new();
+    let mut tests = Vec::new();
+
+    // TODO: iterate in parallel
+    for entry in path.read_dir().wrap_err("retrieving entry")? {
+        let entry = entry?;
+
+        if entry.file_type().wrap_err("retrieving file type")?.is_dir() {
+            suites.push(
+                read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| {
+                    let path = entry.path();
+                    let suite = path.display();
+                    format!("error reading sub-suite {suite}")
+                })?,
+            );
+        } else if entry.file_name().to_string_lossy().contains("_FIXTURE") {
+            continue;
+        } else {
+            let mut test = read_test(entry.path().as_path()).wrap_err_with(|| {
+                let path = entry.path();
+                let suite = path.display();
+                format!("error reading test {suite}")
+            })?;
+
+            if ignore_suite
+                || ignored.contains_any_flag(test.flags)
+                || ignored.contains_test(&test.name)
+                || test
+                    .features
+                    .iter()
+                    .any(|feat| ignored.contains_feature(feat))
+            {
+                test.set_ignored();
+            }
+            tests.push(test);
+        }
+    }
+
+    Ok(TestSuite {
+        name: name.into(),
+        suites: suites.into_boxed_slice(),
+        tests: tests.into_boxed_slice(),
+    })
+}
+
+/// Reads information about a given test case.
+pub(super) fn read_test(path: &Path) -> io::Result<Test> {
+    let name = path
+        .file_stem()
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("test with no file name found: {}", path.display()),
+            )
+        })?
+        .to_str()
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("non-UTF-8 file name found: {}", path.display()),
+            )
+        })?;
+
+    let content = fs::read_to_string(path)?;
+    let metadata = read_metadata(&content, path)?;
+
+    Ok(Test::new(name, content, metadata))
+}
+
+/// Reads the metadata from the input test code.
+fn read_metadata(code: &str, test: &Path) -> io::Result<MetaData> {
+    use once_cell::sync::Lazy;
+    use regex::Regex;
+
+    /// Regular expression to retrieve the metadata of a test.
+    static META_REGEX: Lazy<Regex> = Lazy::new(|| {
+        Regex::new(r#"/\*\-{3}((?:.|\n)*)\-{3}\*/"#)
+            .expect("could not compile metadata regular expression")
+    });
+
+    let yaml = META_REGEX
+        .captures(code)
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("no metadata found for test {}", test.display()),
+            )
+        })?
+        .get(1)
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("no metadata found for test {}", test.display()),
+            )
+        })?
+        .as_str()
+        .replace('\r', "\n");
+
+    serde_yaml::from_str(&yaml).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
+}

javascript-engine/external/boa/boa_tester/src/results.rs

@@ -0,0 +1,518 @@
+use super::SuiteResult;
+use color_eyre::{eyre::WrapErr, Result};
+use serde::{Deserialize, Serialize};
+use std::{
+    env, fs,
+    io::{self, BufReader, BufWriter},
+    path::Path,
+};
+
+/// Structure to store full result information.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+struct ResultInfo {
+    #[serde(rename = "c")]
+    commit: Box<str>,
+    #[serde(rename = "u")]
+    test262_commit: Box<str>,
+    #[serde(rename = "r")]
+    results: SuiteResult,
+}
+
+/// Structure to store full result information.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+struct ReducedResultInfo {
+    #[serde(rename = "c")]
+    commit: Box<str>,
+    #[serde(rename = "u")]
+    test262_commit: Box<str>,
+    #[serde(rename = "t")]
+    total: usize,
+    #[serde(rename = "o")]
+    passed: usize,
+    #[serde(rename = "i")]
+    ignored: usize,
+    #[serde(rename = "p")]
+    panic: usize,
+}
+
+impl From<ResultInfo> for ReducedResultInfo {
+    /// Creates a new reduced suite result from a full suite result.
+    fn from(info: ResultInfo) -> Self {
+        Self {
+            commit: info.commit,
+            test262_commit: info.test262_commit,
+            total: info.results.total,
+            passed: info.results.passed,
+            ignored: info.results.ignored,
+            panic: info.results.panic,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+struct FeaturesInfo {
+    #[serde(rename = "c")]
+    commit: Box<str>,
+    #[serde(rename = "u")]
+    test262_commit: Box<str>,
+    #[serde(rename = "n")]
+    suite_name: Box<str>,
+    #[serde(rename = "f")]
+    features: Vec<String>,
+}
+
+fn remove_duplicates(features_vec: &[String]) -> Vec<String> {
+    let mut result = features_vec.to_vec();
+    result.sort();
+    result.dedup();
+    result
+}
+
+impl From<ResultInfo> for FeaturesInfo {
+    fn from(info: ResultInfo) -> Self {
+        Self {
+            commit: info.commit,
+            test262_commit: info.test262_commit,
+            suite_name: info.results.name,
+            features: remove_duplicates(&info.results.features),
+        }
+    }
+}
+
+/// File name of the "latest results" JSON file.
+const LATEST_FILE_NAME: &str = "latest.json";
+
+/// File name of the "all results" JSON file.
+const RESULTS_FILE_NAME: &str = "results.json";
+
+/// File name of the "features" JSON file.
+const FEATURES_FILE_NAME: &str = "features.json";
+
+/// Writes the results of running the test suite to the given JSON output file.
+///
+/// It will append the results to the ones already present, in an array.
+pub(crate) fn write_json(
+    results: SuiteResult,
+    output: Option<&Path>,
+    verbose: u8,
+) -> io::Result<()> {
+    if let Some(path) = output {
+        let mut branch = env::var("GITHUB_REF").unwrap_or_default();
+        if branch.starts_with("refs/pull") {
+            branch = "pull".to_owned();
+        }
+
+        let path = if branch.is_empty() {
+            path.to_path_buf()
+        } else {
+            let folder = path.join(branch);
+            fs::create_dir_all(&folder)?;
+            folder
+        };
+
+        // We make sure we are using the latest commit information in GitHub pages:
+        update_gh_pages_repo(path.as_path(), verbose);
+
+        if verbose != 0 {
+            println!("Writing the results to {}...", path.display());
+        }
+
+        // Write the latest results.
+
+        let latest_path = path.join(LATEST_FILE_NAME);
+
+        let new_results = ResultInfo {
+            commit: env::var("GITHUB_SHA").unwrap_or_default().into_boxed_str(),
+            test262_commit: get_test262_commit(),
+            results,
+        };
+
+        let latest_output = BufWriter::new(fs::File::create(latest_path)?);
+        serde_json::to_writer(latest_output, &new_results)?;
+
+        // Write the full list of results, retrieving the existing ones first.
+
+        let all_path = path.join(RESULTS_FILE_NAME);
+
+        let mut all_results: Vec<ReducedResultInfo> = if all_path.exists() {
+            serde_json::from_reader(BufReader::new(fs::File::open(&all_path)?))?
+        } else {
+            Vec::new()
+        };
+
+        all_results.push(new_results.clone().into());
+
+        let output = BufWriter::new(fs::File::create(&all_path)?);
+        serde_json::to_writer(output, &all_results)?;
+
+        if verbose != 0 {
+            println!("Results written correctly");
+        }
+
+        // Write the full list of features, existing features go first.
+
+        let features_path = path.join(FEATURES_FILE_NAME);
+
+        let mut all_features: Vec<FeaturesInfo> = if features_path.exists() {
+            serde_json::from_reader(BufReader::new(fs::File::open(&features_path)?))?
+        } else {
+            Vec::new()
+        };
+
+        all_features.push(new_results.into());
+
+        let features_output = BufWriter::new(fs::File::create(&features_path)?);
+        serde_json::to_writer(features_output, &all_features)?;
+
+        if verbose != 0 {
+            println!("Features written correctly");
+        }
+    }
+
+    Ok(())
+}
+
+/// Gets the commit OID of the test262 submodule.
+fn get_test262_commit() -> Box<str> {
+    let mut commit_id = fs::read_to_string(".git/modules/test262/HEAD")
+        .expect("did not find git submodule ref at '.git/modules/test262/HEAD'");
+    // Remove newline.
+    commit_id.pop();
+    commit_id.into_boxed_str()
+}
+
+/// Updates the GitHub pages repository by pulling latest changes before writing the new things.
+fn update_gh_pages_repo(path: &Path, verbose: u8) {
+    if env::var("GITHUB_REF").is_ok() {
+        use std::process::Command;
+
+        // We run the command to pull the gh-pages branch: git -C ../gh-pages/ pull origin
+        Command::new("git")
+            .args(["-C", "../gh-pages", "pull", "--ff-only"])
+            .output()
+            .expect("could not update GitHub Pages");
+
+        // Copy the full results file
+        let from = Path::new("../gh-pages/test262/refs/heads/main/").join(RESULTS_FILE_NAME);
+        let to = path.join(RESULTS_FILE_NAME);
+
+        if verbose != 0 {
+            println!(
+                "Copying the {} file to {} in order to add the results",
+                from.display(),
+                to.display()
+            );
+        }
+
+        fs::copy(from, to).expect("could not copy the main results file");
+    }
+}
+
+/// Compares the results of two test suite runs.
+pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result<()> {
+    let base_results: ResultInfo = serde_json::from_reader(BufReader::new(
+        fs::File::open(base).wrap_err("could not open the base results file")?,
+    ))
+    .wrap_err("could not read the base results")?;
+
+    let new_results: ResultInfo = serde_json::from_reader(BufReader::new(
+        fs::File::open(new).wrap_err("could not open the new results file")?,
+    ))
+    .wrap_err("could not read the new results")?;
+
+    let base_total = base_results.results.total as isize;
+    let new_total = new_results.results.total as isize;
+    let total_diff = new_total - base_total;
+
+    let base_passed = base_results.results.passed as isize;
+    let new_passed = new_results.results.passed as isize;
+    let passed_diff = new_passed - base_passed;
+
+    let base_ignored = base_results.results.ignored as isize;
+    let new_ignored = new_results.results.ignored as isize;
+    let ignored_diff = new_ignored - base_ignored;
+
+    let base_failed = base_total - base_passed - base_ignored;
+    let new_failed = new_total - new_passed - new_ignored;
+    let failed_diff = new_failed - base_failed;
+
+    let base_panics = base_results.results.panic as isize;
+    let new_panics = new_results.results.panic as isize;
+    let panic_diff = new_panics - base_panics;
+
+    let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;
+    let new_conformance = (new_passed as f64 / new_total as f64) * 100_f64;
+    let conformance_diff = new_conformance - base_conformance;
+
+    let test_diff = compute_result_diff(Path::new(""), &base_results.results, &new_results.results);
+
+    if markdown {
+        /// Simple function to add commas as thousands separator for integers.
+        fn pretty_int(i: isize) -> String {
+            let mut res = String::new();
+
+            for (idx, val) in i.abs().to_string().chars().rev().enumerate() {
+                if idx != 0 && idx % 3 == 0 {
+                    res.insert(0, ',');
+                }
+                res.insert(0, val);
+            }
+            res
+        }
+
+        /// Generates a proper diff format, with some bold text if things change.
+        fn diff_format(diff: isize) -> String {
+            format!(
+                "{}{}{}{}",
+                if diff == 0 { "" } else { "**" },
+                if diff.is_positive() {
+                    "+"
+                } else if diff.is_negative() {
+                    "-"
+                } else {
+                    ""
+                },
+                pretty_int(diff),
+                if diff == 0 { "" } else { "**" }
+            )
+        }
+
+        println!("| Test result | main count | PR count | difference |");
+        println!("| :---------: | :----------: | :------: | :--------: |");
+        println!(
+            "| Total | {} | {} | {} |",
+            pretty_int(base_total),
+            pretty_int(new_total),
+            diff_format(total_diff),
+        );
+        println!(
+            "| Passed | {} | {} | {} |",
+            pretty_int(base_passed),
+            pretty_int(new_passed),
+            diff_format(passed_diff),
+        );
+        println!(
+            "| Ignored | {} | {} | {} |",
+            pretty_int(base_ignored),
+            pretty_int(new_ignored),
+            diff_format(ignored_diff),
+        );
+        println!(
+            "| Failed | {} | {} | {} |",
+            pretty_int(base_failed),
+            pretty_int(new_failed),
+            diff_format(failed_diff),
+        );
+        println!(
+            "| Panics | {} | {} | {} |",
+            pretty_int(base_panics),
+            pretty_int(new_panics),
+            diff_format(panic_diff),
+        );
+        println!(
+            "| Conformance | {:.2}% | {:.2}% | {}{}{:.2}%{} |",
+            base_conformance,
+            new_conformance,
+            if conformance_diff.abs() > f64::EPSILON {
+                "**"
+            } else {
+                ""
+            },
+            if conformance_diff > 0_f64 { "+" } else { "" },
+            conformance_diff,
+            if conformance_diff.abs() > f64::EPSILON {
+                "**"
+            } else {
+                ""
+            },
+        );
+
+        if !test_diff.fixed.is_empty() {
+            println!();
+            println!(
+                "<details><summary><b>Fixed tests ({}):</b></summary>",
+                test_diff.fixed.len()
+            );
+            println!("\n```");
+            for test in test_diff.fixed {
+                println!("{test}");
+            }
+            println!("```");
+            println!("</details>");
+        }
+
+        if !test_diff.broken.is_empty() {
+            println!();
+            println!(
+                "<details><summary><b>Broken tests ({}):</b></summary>",
+                test_diff.broken.len()
+            );
+            println!("\n```");
+            for test in test_diff.broken {
+                println!("{test}");
+            }
+            println!("```");
+            println!("</details>");
+        }
+
+        if !test_diff.new_panics.is_empty() {
+            println!();
+            println!(
+                "<details><summary><b>New panics ({}):</b></summary>",
+                test_diff.new_panics.len()
+            );
+            println!("\n```");
+            for test in test_diff.new_panics {
+                println!("{test}");
+            }
+            println!("```");
+            println!("</details>");
+        }
+
+        if !test_diff.panic_fixes.is_empty() {
+            println!();
+            println!(
+                "<details><summary><b>Fixed panics ({}):</b></summary>",
+                test_diff.panic_fixes.len()
+            );
+            println!("\n```");
+            for test in test_diff.panic_fixes {
+                println!("{test}");
+            }
+            println!("```");
+            println!("</details>");
+        }
+    } else {
+        println!("Test262 conformance changes:");
+        println!("| Test result | main |    PR   | difference |");
+        println!(
+            "|    Passed   | {base_passed:^6} | {new_passed:^5} | {:^10} |",
+            base_passed - new_passed
+        );
+        println!(
+            "|   Ignored   | {base_ignored:^6} | {new_ignored:^5} | {:^10} |",
+            base_ignored - new_ignored
+        );
+        println!(
+            "|   Failed    | {base_failed:^6} | {new_failed:^5} | {:^10} |",
+            base_failed - new_failed,
+        );
+        println!(
+            "|   Panics    | {base_panics:^6} | {new_panics:^5} | {:^10} |",
+            base_panics - new_panics
+        );
+
+        if !test_diff.fixed.is_empty() {
+            println!();
+            println!("Fixed tests ({}):", test_diff.fixed.len());
+            for test in test_diff.fixed {
+                println!("{test}");
+            }
+        }
+
+        if !test_diff.broken.is_empty() {
+            println!();
+            println!("Broken tests ({}):", test_diff.broken.len());
+            for test in test_diff.broken {
+                println!("{test}");
+            }
+        }
+
+        if !test_diff.new_panics.is_empty() {
+            println!();
+            println!("New panics ({}):", test_diff.new_panics.len());
+            for test in test_diff.new_panics {
+                println!("{test}");
+            }
+        }
+
+        if !test_diff.panic_fixes.is_empty() {
+            println!();
+            println!("Fixed panics ({}):", test_diff.panic_fixes.len());
+            for test in test_diff.panic_fixes {
+                println!("{test}");
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Test differences.
+#[derive(Debug, Clone, Default)]
+struct ResultDiff {
+    fixed: Vec<Box<str>>,
+    broken: Vec<Box<str>>,
+    new_panics: Vec<Box<str>>,
+    panic_fixes: Vec<Box<str>>,
+}
+
+impl ResultDiff {
+    /// Extends the diff with new results.
+    fn extend(&mut self, new: Self) {
+        self.fixed.extend(new.fixed);
+        self.broken.extend(new.broken);
+        self.new_panics.extend(new.new_panics);
+        self.panic_fixes.extend(new.panic_fixes);
+    }
+}
+
+/// Compares a base and a new result and returns the list of differences.
+fn compute_result_diff(
+    base: &Path,
+    base_result: &SuiteResult,
+    new_result: &SuiteResult,
+) -> ResultDiff {
+    use super::TestOutcomeResult;
+
+    let mut final_diff = ResultDiff::default();
+
+    for base_test in &base_result.tests {
+        if let Some(new_test) = new_result
+            .tests
+            .iter()
+            .find(|new_test| new_test.strict == base_test.strict && new_test.name == base_test.name)
+        {
+            let test_name = format!(
+                "test/{}/{}.js {}(previously {:?})",
+                base.display(),
+                new_test.name,
+                if base_test.strict {
+                    "[strict mode] "
+                } else {
+                    ""
+                },
+                base_test.result
+            )
+            .into_boxed_str();
+
+            #[allow(clippy::match_same_arms)]
+            match (base_test.result, new_test.result) {
+                (a, b) if a == b => {}
+                (TestOutcomeResult::Ignored, TestOutcomeResult::Failed) => {}
+
+                (_, TestOutcomeResult::Passed) => final_diff.fixed.push(test_name),
+                (TestOutcomeResult::Panic, _) => final_diff.panic_fixes.push(test_name),
+                (_, TestOutcomeResult::Failed) => final_diff.broken.push(test_name),
+                (_, TestOutcomeResult::Panic) => final_diff.new_panics.push(test_name),
+
+                _ => {}
+            }
+        }
+    }
+
+    for base_suite in &base_result.suites {
+        if let Some(new_suite) = new_result
+            .suites
+            .iter()
+            .find(|new_suite| new_suite.name == base_suite.name)
+        {
+            let new_base = base.join(new_suite.name.as_ref());
+            let diff = compute_result_diff(new_base.as_path(), base_suite, new_suite);
+
+            final_diff.extend(diff);
+        }
+    }
+
+    final_diff
+}