diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 83fdc540..76cf9861 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -40,10 +40,9 @@ globalThis.dumpJSONResults ??= false; globalThis.testList ??= undefined; globalThis.startDelay ??= undefined; globalThis.shouldReport ??= false; +globalThis.prefetchResources ??= true; function getIntParam(urlParams, key) { - if (!urlParams.has(key)) - return undefined const rawValue = urlParams.get(key); const value = parseInt(rawValue); if (value <= 0) @@ -51,6 +50,11 @@ function getIntParam(urlParams, key) { return value; } +function getBoolParam(urlParams, key) { + const rawValue = urlParams.get(key).toLowerCase() + return !(rawValue === "false" || rawValue === "0") + } + function getTestListParam(urlParams, key) { if (globalThis.testList?.length) throw new Error(`Overriding previous testList=${globalThis.testList.join()} with ${key} url-parameter.`); @@ -73,8 +77,13 @@ if (typeof(URLSearchParams) !== "undefined") { globalThis.testIterationCount = getIntParam(urlParameters, "iterationCount"); if (urlParameters.has("worstCaseCount")) globalThis.testWorstCaseCount = getIntParam(urlParameters, "worstCaseCount"); + if (urlParameters.has("prefetchResources")) + globalThis.prefetchResources = getBoolParam(urlParameters, "prefetchResources"); } +if (!globalThis.prefetchResources) + console.warn("Disabling resource prefetching!"); + // Used for the promise representing the current benchmark run. this.currentResolve = null; this.currentReject = null; @@ -88,7 +97,7 @@ function displayCategoryScores() { let summaryElement = document.getElementById("result-summary"); for (let [category, scores] of categoryScores) - summaryElement.innerHTML += `

${category}: ${uiFriendlyScore(geomean(scores))}

` + summaryElement.innerHTML += `

${category}: ${uiFriendlyScore(geomeanScore(scores))}

` categoryScores = null; } @@ -138,12 +147,15 @@ function mean(values) { return sum / values.length; } -function geomean(values) { +function geomeanScore(values) { assert(values instanceof Array); let product = 1; for (let x of values) product *= x; - return product ** (1 / values.length); + const score = product ** (1 / values.length); + // Allow 0 for uninitialized subScores(). + assert(score >= 0, `Got invalid score: ${score}`) + return score; } function toScore(timeValue) { @@ -180,28 +192,29 @@ function uiFriendlyDuration(time) { // TODO: Cleanup / remove / merge. This is only used for caching loads in the // non-browser setting. In the browser we use exclusively `loadCache`, // `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below. -const fileLoader = (function() { - class Loader { - constructor() { - this.requests = new Map; - } - - // Cache / memoize previously read files, because some workloads - // share common code. - load(url) { - assert(!isInBrowser); +class ShellFileLoader { + constructor() { + this.requests = new Map; + } - if (this.requests.has(url)) { - return this.requests.get(url); - } + // Cache / memoize previously read files, because some workloads + // share common code. + load(url) { + assert(!isInBrowser); + if (!globalThis.prefetchResources) + return `load("${url}");` - const contents = readFile(url); - this.requests.set(url, contents); - return contents; + if (this.requests.has(url)) { + return this.requests.get(url); } + + const contents = readFile(url); + this.requests.set(url, contents); + return contents; } - return new Loader; -})(); +}; + +const shellFileLoader = new ShellFileLoader(); class Driver { constructor(benchmarks) { @@ -211,6 +224,7 @@ class Driver { // Make benchmark list unique and sort it. this.benchmarks = Array.from(new Set(benchmarks)); this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1); + assert(this.benchmarks.length, "No benchmarks selected"); // TODO: Cleanup / remove / merge `blobDataCache` and `loadCache` vs. // the global `fileLoader` cache. this.blobDataCache = { }; @@ -248,7 +262,7 @@ class Driver { performance.mark("update-ui"); benchmark.updateUIAfterRun(); - if (isInBrowser) { + if (isInBrowser && globalThis.prefetchResources) { const cache = JetStream.blobDataCache; for (const file of benchmark.plan.files) { const blobData = cache[file]; @@ -270,8 +284,11 @@ class Driver { } const allScores = []; - for (const benchmark of this.benchmarks) - allScores.push(benchmark.score); + for (const benchmark of this.benchmarks) { + const score = benchmark.score; + assert(score > 0, `Invalid ${benchmark.name} score: ${score}`); + allScores.push(score); + } categoryScores = new Map; for (const benchmark of this.benchmarks) { @@ -282,23 +299,27 @@ class Driver { for (const benchmark of this.benchmarks) { for (let [category, value] of Object.entries(benchmark.subScores())) { const arr = categoryScores.get(category); + assert(value > 0, `Invalid ${benchmark.name} ${category} score: ${value}`); arr.push(value); } } + const totalScore = geomeanScore(allScores); + assert(totalScore > 0, `Invalid total score: ${totalScore}`); + if (isInBrowser) { - summaryElement.classList.add('done'); - summaryElement.innerHTML = `
${uiFriendlyScore(geomean(allScores))}
`; + summaryElement.classList.add("done"); + summaryElement.innerHTML = `
${uiFriendlyScore(totalScore)}
`; summaryElement.onclick = displayCategoryScores; if (showScoreDetails) displayCategoryScores(); - statusElement.innerHTML = ''; + statusElement.innerHTML = ""; } else if (!dumpJSONResults) { console.log("\n"); for (let [category, scores] of categoryScores) - console.log(`${category}: ${uiFriendlyScore(geomean(scores))}`); + console.log(`${category}: ${uiFriendlyScore(geomeanScore(scores))}`); - console.log("\nTotal Score: ", uiFriendlyScore(geomean(allScores)), "\n"); + console.log("\nTotal Score: ", uiFriendlyScore(totalScore), "\n"); } this.reportScoreToRunBenchmarkRunner(); @@ -727,7 +748,7 @@ class Benchmark { get score() { const subScores = Object.values(this.subScores()); - return geomean(subScores); + return geomeanScore(subScores); } subScores() { @@ -788,8 +809,9 @@ class Benchmark { scripts.add(text); } else { const cache = JetStream.blobDataCache; - for (const file of this.plan.files) - scripts.addWithURL(cache[file].blobURL); + for (const file of this.plan.files) { + scripts.addWithURL(globalThis.prefetchResources ? cache[file].blobURL : file); + } } const promise = new Promise((resolve, reject) => { @@ -838,6 +860,11 @@ class Benchmark { } async doLoadBlob(resource) { + const blobData = JetStream.blobDataCache[resource]; + if (!globalThis.prefetchResources) { + blobData.blobURL = resource; + return blobData; + } let response; let tries = 3; while (tries--) { @@ -854,7 +881,6 @@ class Benchmark { throw new Error("Fetch failed"); } const blob = await response.blob(); - const blobData = JetStream.blobDataCache[resource]; blobData.blob = blob; blobData.blobURL = URL.createObjectURL(blob); return blobData; @@ -987,7 +1013,7 @@ class Benchmark { assert(!isInBrowser); assert(this.scripts === null, "This initialization should be called only once."); - this.scripts = this.plan.files.map(file => fileLoader.load(file)); + this.scripts = this.plan.files.map(file => shellFileLoader.load(file)); assert(this.preloads === null, "This initialization should be called only once."); this.preloads = Object.entries(this.plan.preload ?? {}); @@ -2417,8 +2443,7 @@ for (const benchmark of BENCHMARKS) { } -function processTestList(testList) -{ +function processTestList(testList) { let benchmarkNames = []; let benchmarks = []; @@ -2430,7 +2455,7 @@ function processTestList(testList) for (let name of benchmarkNames) { name = name.toLowerCase(); if (benchmarksByTag.has(name)) - benchmarks.concat(findBenchmarksByTag(name)); + benchmarks = benchmarks.concat(findBenchmarksByTag(name)); else benchmarks.push(findBenchmarkByName(name)); } diff --git a/cli.js b/cli.js index 612a7b68..b670d21e 100644 --- a/cli.js +++ b/cli.js @@ -56,6 +56,8 @@ if (typeof runMode !== "undefined" && runMode == "RAMification") globalThis.RAMification = true; if ("--ramification" in cliFlags) globalThis.RAMification = true; +if ("--no-prefetch" in cliFlags) + globalThis.prefetchResources = false; if (cliArgs.length) globalThis.testList = cliArgs; @@ -78,9 +80,11 @@ if ("--help" in cliFlags) { console.log(""); console.log("Options:"); - console.log(" --iteration-count: Set the default iteration count."); - console.log(" --worst-case-count: Set the default worst-case count"); + console.log(" --iteration-count: Set the default iteration count."); + console.log(" --worst-case-count: Set the default worst-case count"); console.log(" --dump-json-results: Print summary json to the console."); + console.log(" --ramification: Enable ramification support. See RAMification.py for more details."); + console.log(" --no-prefetch: Do not prefetch resources. Will add network overhead to measurements!"); console.log(""); console.log("Available tags:"); diff --git a/package.json b/package.json index efc89908..7760f4b9 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test:firefox": "node tests/run.mjs --browser firefox", "test:safari": "node tests/run.mjs --browser safari", "test:edge": "node tests/run.mjs --browser edge", + "test:shell": "npm run test:v8 && npm run test:jsc && npm run test:spidermonkey", "test:v8": "node tests/run-shell.mjs --shell v8", "test:jsc": "node tests/run-shell.mjs --shell jsc", "test:spidermonkey": "node tests/run-shell.mjs --shell spidermonkey" diff --git a/shell-config.js b/shell-config.js index 4c88186c..8b29c7bf 100644 --- a/shell-config.js +++ b/shell-config.js @@ -27,6 +27,7 @@ const isInBrowser = false; console = { log: globalThis?.console?.log ?? print, error: globalThis?.console?.error ?? print, + warn: globalThis?.console?.warn ?? print, } const isD8 = typeof Realm !== "undefined"; diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index f31a646b..1b626a7d 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -1,7 +1,7 @@ #! /usr/bin/env node import commandLineArgs from "command-line-args"; -import { spawnSync } from "child_process"; +import { spawn } from "child_process"; import { fileURLToPath } from "url"; import { styleText } from "node:util"; import * as path from "path"; @@ -59,7 +59,7 @@ const SPAWN_OPTIONS = { stdio: ["inherit", "inherit", "inherit"] }; -function sh(binary, ...args) { +async function sh(binary, ...args) { const cmd = `${binary} ${args.join(" ")}`; if (GITHUB_ACTIONS_OUTPUT) { core.startGroup(binary); @@ -68,22 +68,48 @@ function sh(binary, ...args) { console.log(styleText("blue", cmd)); } try { - const result = spawnSync(binary, args, SPAWN_OPTIONS); + const result = await spawnCaptureStdout(binary, args, SPAWN_OPTIONS); if (result.status || result.error) { logError(result.error); throw new Error(`Shell CMD failed: ${binary} ${args.join(" ")}`); } + return result; } finally { if (GITHUB_ACTIONS_OUTPUT) core.endGroup(); } } +async function spawnCaptureStdout(binary, args) { + const childProcess = spawn(binary, args); + childProcess.stdout.pipe(process.stdout); + return new Promise((resolve, reject) => { + childProcess.stdoutString = ""; + childProcess.stdio[1].on("data", (data) => { + childProcess.stdoutString += data.toString(); + }); + childProcess.on('close', (code) => { + if (code === 0) { + resolve(childProcess); + } else { + // Reject the Promise with an Error on failure + const error = new Error(`Command failed with exit code ${code}: ${binary} ${args.join(" ")}`); + error.process = childProcess; + error.stdout = childProcess.stdoutString; + error.exitCode = code; + reject(error); + } + }); + childProcess.on('error', reject); + }) +} + async function runTests() { const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); let success = true; success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH)); success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx"); + success &&= await runCLITest("Run Tag No Prefetch", shellBinary, "proxy", "--no-prefetch"); success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled"); success &&= await runCLITest("Run Default Suite", shellBinary); if (!success) @@ -111,8 +137,8 @@ function jsvuOSName() { const DEFAULT_JSC_LOCATION = "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc" -function testSetup() { - sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`); +async function testSetup() { + await sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`); let shellBinary = path.join(os.homedir(), ".jsvu/bin", SHELL_NAME); if (!fs.existsSync(shellBinary) && SHELL_NAME == "javascriptcore") shellBinary = DEFAULT_JSC_LOCATION; @@ -123,7 +149,13 @@ function testSetup() { } function runCLITest(name, shellBinary, ...args) { - return runTest(name, () => sh(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); + return runTest(name, () => runShell(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); +} + +async function runShell(shellBinary, ...args) { + const result = await sh(shellBinary, ...args); + if (result.stdoutString.includes("JetStream3 failed")) + throw new Error("test failed") } setImmediate(runTests); diff --git a/tests/run.mjs b/tests/run.mjs index 67f0f11f..0b9f5959 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -60,9 +60,10 @@ const server = await serve(PORT); async function runTests() { let success = true; try { - success &&= await runTest("Run Single Suite", () => testEnd2End({ test: "proxy-mobx" })); - success &&= await runTest("Run Disabled Suite", () => testEnd2End({ tag: "disabled" })); - success &&= await runTest("Run Default Suite", () => testEnd2End()); + success &&= await runEnd2EndTest("Run Single Suite", { test: "proxy-mobx" }); + success &&= await runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); + success &&= await runEnd2EndTest("Run Disabled Suite", { tag: "disabled" }); + success &&= await runEnd2EndTest("Run Default Suite"); } finally { server.close(); } @@ -70,6 +71,9 @@ async function runTests() { process.exit(1); } +async function runEnd2EndTest(name, params) { + return runTest(name, () => testEnd2End(params)); +} async function testEnd2End(params) { const driver = await new Builder().withCapabilities(capabilities).build();