From 1cfcdf8a1d08ce077e1107128ecd5bd2e61d304f Mon Sep 17 00:00:00 2001 From: Nathan Houle Date: Tue, 25 Mar 2025 11:42:11 -0700 Subject: [PATCH 1/2] test(e2e): isolate E2E tests and remove destructive actions The E2E suite has a litany of problems, including that it is fairly complex, doesn't work very well locally and destructively modifies the workspace, and doesn't isolate tests from each other particularly well. The tests rely on wrapper scripts to work properly, which makes it unnecessarily difficult to run E2E tests. This changeset makes a few changes to the E2E test to help address these issues The tl;dr is that tests are now self contained without any need for wrapper scripts and are better isolated. Other notable changes include: - Each test now gets its own isolated Verdaccio registry; the Verdaccio registry storage is no longer shared between tests and is no longer persisted between test invocations. - The CLI is now published to the (isolated) Verdaccio registry from a temporary workspace, which is a copy of the project workspace. We no longer publish to Verdaccio from the project workspace. This ensures that destructive actions that occur on publish don't alter the workspace, and ensures that one test can't modify the registry in a way that impacts another test. - The tests no longer rely on wrapper scripts (which ran `vitest` in a subprocess and didn't always work locally (depending on your `PATH`). - I also converted the tests to TypeScript while I was at it. Theoretically, these changes mean we can run E2E tests concurrently, though I haven't done that in this changeset. We could also now merge the E2E vitest configuration into the primary Vitest config (though I haven't done that here, either). --- e2e/install.e2e.js | 44 ---------- e2e/install.e2e.ts | 198 +++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 140 ++++++++++++++++++++---------- package.json | 7 +- tools/e2e/run.js | 41 --------- tools/e2e/setup.js | 166 ------------------------------------ vitest.e2e.config.ts | 2 +- 7 files changed, 299 insertions(+), 299 deletions(-) delete mode 100644 e2e/install.e2e.js create mode 100644 e2e/install.e2e.ts delete mode 100644 tools/e2e/run.js delete mode 100644 tools/e2e/setup.js diff --git a/e2e/install.e2e.js b/e2e/install.e2e.js deleted file mode 100644 index e5204eac41b..00000000000 --- a/e2e/install.e2e.js +++ /dev/null @@ -1,44 +0,0 @@ -import { readFileSync, existsSync } from 'fs' -import { mkdir } from 'fs/promises' -import { platform } from 'os' -import { join, resolve } from 'path' -import { env } from 'process' -import { fileURLToPath } from 'url' - -import execa from 'execa' -import { expect, test } from 'vitest' - -import { packageManagerConfig, packageManagerExists } from './utils.js' - -const { version } = JSON.parse(readFileSync(fileURLToPath(new URL('../package.json', import.meta.url)), 'utf-8')) - -/** - * Prepares the workspace for the test suite to run - * @param {string} folderName - */ -const prepare = async (folderName) => { - const folder = join(env.E2E_TEST_WORKSPACE, folderName) - await mkdir(folder, { recursive: true }) - return folder -} - -Object.entries(packageManagerConfig).forEach(([packageManager, { install: installCmd, lockFile }]) => { - test.runIf(packageManagerExists(packageManager))( - `${packageManager} → should install the cli and run the help command`, - async () => { - const cwd = await prepare(`${packageManager}-try-install`) - await execa(...installCmd, { stdio: env.DEBUG ? 'inherit' : 'ignore', cwd }) - - expect(existsSync(join(cwd, lockFile)), `Generated lock file ${lockFile} does not exists in ${cwd}`).toBe(true) - - const binary = resolve(join(cwd, `./node_modules/.bin/netlify${platform() === 'win32' ? '.cmd' : ''}`)) - const { stdout } = await execa(binary, ['help'], { cwd }) - - expect(stdout.trim(), `Help command does not start with 'VERSION':\n\n${stdout}`).toMatch(/^VERSION/) - expect(stdout, `Help command does not include 'netlify-cli/${version}':\n\n${stdout}`).toContain( - `netlify-cli/${version}`, - ) - expect(stdout, `Help command does not include '$ netlify [COMMAND]':\n\n${stdout}`).toMatch('$ netlify [COMMAND]') - }, - ) -}) diff --git a/e2e/install.e2e.ts b/e2e/install.e2e.ts new file mode 100644 index 00000000000..2506bd549f6 --- /dev/null +++ b/e2e/install.e2e.ts @@ -0,0 +1,198 @@ +import http from 'node:http' +import os from 'node:os' +import events from 'node:events' +import { existsSync } from 'node:fs' +import { execSync } from 'node:child_process' +import fs from 'node:fs/promises' +import { platform } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import execa from 'execa' +import { runServer } from 'verdaccio' +import { describe, expect, it } from 'vitest' +import createDebug from 'debug' +import picomatch from 'picomatch' + +import pkg from '../package.json' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const projectRoot = path.resolve(__dirname, '..') +const distDir = path.join(projectRoot, 'dist') +const tempdirPrefix = 'netlify-cli-e2e-test--' + +const debug = createDebug('netlify-cli:e2e') +const isNodeModules = picomatch('**/node_modules/**') +const isNotNodeModules = (target: string) => !isNodeModules(target) + +const itWithMockNpmRegistry = it.extend<{ registry: { address: string; cwd: string } }>({ + registry: async ( + // Vitest requires this argument is destructured even if no properties are used + // eslint-disable-next-line no-empty-pattern + {}, + use, + ) => { + try { + if (!(await fs.stat(distDir)).isDirectory()) { + throw new Error(`found unexpected non-directory at "${distDir}"`) + } + } catch (err) { + throw new Error( + '"dist" directory does not exist or is not a directory. The project must be built before running E2E tests.', + { cause: err }, + ) + } + + const verdaccioStorageDir = await fs.mkdtemp(path.join(os.tmpdir(), `${tempdirPrefix}verdaccio-storage`)) + const server: http.Server = (await runServer( + // @ts-expect-error(ndhoule): Verdaccio's types are incorrect + { + self_path: __dirname, + storage: verdaccioStorageDir, + web: { title: 'Test Registry' }, + max_body_size: '128mb', + // Disable user registration + max_users: -1, + logs: { level: 'fatal' }, + uplinks: { + npmjs: { + url: 'https://registry.npmjs.org/', + maxage: '1d', + cache: true, + }, + }, + packages: { + '@*/*': { + access: '$all', + publish: 'noone', + proxy: 'npmjs', + }, + 'netlify-cli': { + access: '$all', + publish: '$all', + }, + '**': { + access: '$all', + publish: 'noone', + proxy: 'npmjs', + }, + }, + }, + )) as http.Server + + await Promise.all([ + Promise.race([ + events.once(server, 'listening'), + events.once(server, 'error').then(() => { + throw new Error('Verdaccio server failed to start') + }), + ]), + server.listen(), + ]) + const address = server.address() + if (address === null || typeof address === 'string') { + throw new Error('Failed to open Verdaccio server') + } + const registryURL = new URL( + `http://${ + address.family === 'IPv6' && address.address === '::' ? 'localhost' : address.address + }:${address.port.toString()}`, + ) + + // The CLI publishing process modifies the workspace, so copy it to a temporary directory. This + // lets us avoid contaminating the user's workspace when running these tests locally. + const publishWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), `${tempdirPrefix}publish-workspace`)) + await fs.cp(projectRoot, publishWorkspace, { + recursive: true, + verbatimSymlinks: true, + // At this point, the project is built. As long as we limit the prepublish script to built- + // ins, node_modules are not be necessary to publish the package. + filter: isNotNodeModules, + }) + await fs.writeFile( + path.join(publishWorkspace, '.npmrc'), + `//${registryURL.hostname}:${registryURL.port}/:_authToken=dummy`, + ) + await execa('npm', ['publish', `--registry=${registryURL.toString()}`, '--tag=testing'], { + cwd: publishWorkspace, + stdio: debug.enabled ? 'inherit' : 'ignore', + }) + await fs.rm(publishWorkspace, { force: true, recursive: true }) + + const testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), tempdirPrefix)) + await use({ + address: registryURL.toString(), + cwd: testWorkspace, + }) + + await Promise.all([ + events.once(server, 'close'), + server.close(), + // eslint-disable-next-line @typescript-eslint/no-confusing-void-expression + server.closeAllConnections(), + ]) + await fs.rm(testWorkspace, { force: true, recursive: true }) + await fs.rm(verdaccioStorageDir, { force: true, recursive: true }) + }, +}) + +const doesPackageManagerExist = (packageManager: string): boolean => { + try { + execSync(`${packageManager} --version`) + return true + } catch { + return false + } +} + +const tests: [packageManager: string, config: { install: [cmd: string, args: string[]]; lockfile: string }][] = [ + [ + 'npm', + { + install: ['npm', ['install', 'netlify-cli@testing']], + lockfile: 'package-lock.json', + }, + ], + [ + 'pnpm', + { + install: ['pnpm', ['add', 'netlify-cli@testing']], + lockfile: 'pnpm-lock.yaml', + }, + ], + [ + 'yarn', + { + install: ['yarn', ['add', 'netlify-cli@testing']], + lockfile: 'yarn.lock', + }, + ], +] + +describe.each(tests)('%s → installs the cli and runs the help command without error', (packageManager, config) => { + itWithMockNpmRegistry.runIf(doesPackageManagerExist(packageManager))( + 'installs the cli and runs the help command without error', + async ({ registry }) => { + const cwd = registry.cwd + await execa(...config.install, { + cwd, + env: { npm_config_registry: registry.address }, + stdio: debug.enabled ? 'inherit' : 'ignore', + }) + + expect( + existsSync(path.join(cwd, config.lockfile)), + `Generated lock file ${config.lockfile} does not exist in ${cwd}`, + ).toBe(true) + + const binary = path.resolve(path.join(cwd, `./node_modules/.bin/netlify${platform() === 'win32' ? '.cmd' : ''}`)) + const { stdout } = await execa(binary, ['help'], { cwd }) + + expect(stdout.trim(), `Help command does not start with 'VERSION':\n\n${stdout}`).toMatch(/^VERSION/) + expect(stdout, `Help command does not include 'netlify-cli/${pkg.version}':\n\n${stdout}`).toContain( + `netlify-cli/${pkg.version}`, + ) + expect(stdout, `Help command does not include '$ netlify [COMMAND]':\n\n${stdout}`).toMatch('$ netlify [COMMAND]') + }, + ) +}) diff --git a/package-lock.json b/package-lock.json index e93c294fdd9..4044060c9bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -143,6 +143,7 @@ "@types/parallel-transform": "1.1.4", "@types/parse-github-url": "1.0.3", "@types/parse-gitignore": "1.0.2", + "@types/picomatch": "^3.0.2", "@types/prettyjson": "0.0.33", "@types/semver": "7.5.8", "@types/serialize-javascript": "^5.0.4", @@ -163,6 +164,7 @@ "nock": "14.0.1", "npm-run-all2": "^7.0.2", "p-timeout": "6.1.4", + "picomatch": "^4.0.2", "prettier": "2.6.2", "serialize-javascript": "6.0.2", "strip-ansi": "7.1.0", @@ -2236,19 +2238,6 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, - "node_modules/@netlify/build/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@netlify/build/node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -2347,6 +2336,18 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/@netlify/cache-utils/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/@netlify/cache-utils/node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -4579,17 +4580,6 @@ } } }, - "node_modules/@rollup/pluginutils/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.31.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.31.0.tgz", @@ -5230,6 +5220,13 @@ "@types/node": "*" } }, + "node_modules/@types/picomatch": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-3.0.2.tgz", + "integrity": "sha512-n0i8TD3UDB7paoMMxA3Y65vUncFJXjcUf7lQY7YyKGl6031FNjfsLs6pdLFCy2GNFxItPJG8GvvpbZc2skH7WA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/prettyjson": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/@types/prettyjson/-/prettyjson-0.0.33.tgz", @@ -7160,6 +7157,18 @@ "node": ">= 8" } }, + "node_modules/anymatch/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/apache-md5": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/apache-md5/-/apache-md5-1.1.8.tgz", @@ -8486,6 +8495,18 @@ "fsevents": "~2.3.2" } }, + "node_modules/chokidar/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/chokidar/node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -14471,6 +14492,18 @@ "node": ">=8.6" } }, + "node_modules/micromatch/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -15706,11 +15739,12 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "license": "MIT", "engines": { - "node": ">=8.6" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" @@ -21662,13 +21696,6 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==" }, - "picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "optional": true, - "peer": true - }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -21829,6 +21856,11 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==" }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -23118,13 +23150,6 @@ "@types/estree": "^1.0.0", "estree-walker": "^2.0.2", "picomatch": "^4.0.2" - }, - "dependencies": { - "picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==" - } } }, "@rollup/rollup-android-arm-eabi": { @@ -23617,6 +23642,12 @@ "@types/node": "*" } }, + "@types/picomatch": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-3.0.2.tgz", + "integrity": "sha512-n0i8TD3UDB7paoMMxA3Y65vUncFJXjcUf7lQY7YyKGl6031FNjfsLs6pdLFCy2GNFxItPJG8GvvpbZc2skH7WA==", + "dev": true + }, "@types/prettyjson": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/@types/prettyjson/-/prettyjson-0.0.33.tgz", @@ -24951,6 +24982,13 @@ "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" + }, + "dependencies": { + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + } } }, "apache-md5": { @@ -25871,6 +25909,11 @@ "readdirp": "~3.6.0" }, "dependencies": { + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -30113,6 +30156,13 @@ "requires": { "braces": "^3.0.3", "picomatch": "^2.3.1" + }, + "dependencies": { + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + } } }, "mime": { @@ -30970,9 +31020,9 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==" }, "pidtree": { "version": "0.6.0", diff --git a/package.json b/package.json index 331488e9b06..570d8de7110 100644 --- a/package.json +++ b/package.json @@ -31,12 +31,13 @@ "url": "https://github.com/netlify/cli/issues" }, "scripts": { - "_format": "prettier --loglevel=warn \"{src,tools,scripts,tests,.github}/**/*.{mjs,cjs,js,mts,md,yml,json,html,ts}\" \"*.{mjs,cjs,js,mts,yml,json,html,ts}\" \".*.{mjs,cjs,js,yml,json,html,ts}\" \"!CHANGELOG.md\" \"!**/*/package-lock.json\" \"!.github/**/*.md\"", + "_format": "prettier --loglevel=warn \"{src,tools,scripts,tests,e2e,.github}/**/*.{mjs,cjs,js,mts,md,yml,json,html,ts}\" \"*.{mjs,cjs,js,mts,yml,json,html,ts}\" \".*.{mjs,cjs,js,yml,json,html,ts}\" \"!CHANGELOG.md\" \"!**/*/package-lock.json\" \"!.github/**/*.md\"", "build": "tsc --project tsconfig.build.json", "clean": "rm -rf dist/", "dev": "tsc --project tsconfig.build.json --watch", "docs": "npm run --prefix=site build", - "e2e": "node ./tools/e2e/run.js", + "e2e": "vitest run --config vitest.e2e.config.ts", + "e2e:debug": "DEBUG='netlify-cli:test' npm run e2e", "format": "npm run _format -- --write", "format:check": "npm run _format -- --check", "start": "node ./bin/run.js", @@ -190,6 +191,7 @@ "@types/parallel-transform": "1.1.4", "@types/parse-github-url": "1.0.3", "@types/parse-gitignore": "1.0.2", + "@types/picomatch": "^3.0.2", "@types/prettyjson": "0.0.33", "@types/semver": "7.5.8", "@types/serialize-javascript": "^5.0.4", @@ -210,6 +212,7 @@ "nock": "14.0.1", "npm-run-all2": "^7.0.2", "p-timeout": "6.1.4", + "picomatch": "^4.0.2", "prettier": "2.6.2", "serialize-javascript": "6.0.2", "strip-ansi": "7.1.0", diff --git a/tools/e2e/run.js b/tools/e2e/run.js deleted file mode 100644 index 5f5c9540f59..00000000000 --- a/tools/e2e/run.js +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env node -import { dirname, join } from 'path' -import { exit } from 'process' -import { fileURLToPath } from 'url' - -import execa from 'execa' - -import { setup } from './setup.js' - -/** The main test runner function */ -const main = async () => { - const { cleanup, registry, workspace } = await setup() - - // By default assume it is failing, so we don't have to set it when something goes wrong - // if it is going successful it will be set - let statusCode = 1 - - try { - console.log('Start running tests for ./e2e/**/*.e2e.js') - const { exitCode } = await execa('vitest', ['run', '--config=vitest.e2e.config.ts', '--reporter=basic'], { - stdio: 'inherit', - cwd: join(dirname(fileURLToPath(import.meta.url)), '../..'), - env: { - E2E_TEST_WORKSPACE: workspace, - E2E_TEST_REGISTRY: registry, - }, - }) - statusCode = exitCode - } catch (error_) { - await cleanup() - console.error(error_ instanceof Error ? error_.message : error_) - } - - await cleanup() - exit(statusCode) -} - -main().catch((error_) => { - console.error(error_ instanceof Error ? error_.message : error_) - exit(1) -}) diff --git a/tools/e2e/setup.js b/tools/e2e/setup.js deleted file mode 100644 index 628b368c4c3..00000000000 --- a/tools/e2e/setup.js +++ /dev/null @@ -1,166 +0,0 @@ -import { appendFile, mkdtemp, readFile, rm, writeFile } from 'fs/promises' -import { tmpdir } from 'os' -import { dirname, join, normalize, sep } from 'path' -import { env } from 'process' -import { fileURLToPath } from 'url' - -import execa from 'execa' -import getPort from 'get-port' -import pTimeout from 'p-timeout' -import { runServer } from 'verdaccio' - -import { fileExistsAsync } from '../../dist/lib/fs.js' - -const dir = dirname(fileURLToPath(import.meta.url)) -const rootDir = normalize(join(dir, '../..')) - -const VERDACCIO_TIMEOUT_MILLISECONDS = 60 * 1000 -const START_PORT_RANGE = 5000 -const END_PORT_RANGE = 5000 - -/** - * Gets the verdaccio configuration - */ -const getVerdaccioConfig = () => ({ - // workaround - // on v5 the `self_path` still exists and will be removed in v6 of verdaccio - self_path: dir, - storage: normalize(join(rootDir, '.verdaccio-storage')), - web: { title: 'Test Registry' }, - max_body_size: '128mb', - // Disable creation of users this is only meant for integration testing - // where it should not be necessary to authenticate. Therefore no user is needed - max_users: -1, - logs: { level: 'fatal' }, - uplinks: { - npmjs: { - url: 'https://registry.npmjs.org/', - maxage: '1d', - cache: true, - }, - }, - packages: { - '@*/*': { - access: '$all', - publish: 'noone', - proxy: 'npmjs', - }, - 'netlify-cli': { - access: '$all', - publish: '$all', - }, - '**': { - access: '$all', - publish: 'noone', - proxy: 'npmjs', - }, - }, -}) - -/** - * Start verdaccio server - * @returns {Promise<{ url: URL; storage: string; }>} - */ -const runVerdaccio = async (config, port) => { - const app = await runServer(config) - - return new Promise((resolve, reject) => { - app.listen(port, 'localhost', () => { - resolve({ url: new URL(`http://localhost:${port}/`), storage: config.storage }) - }) - app.on('error', (error) => { - reject(error) - }) - }) -} - -/** - * Start verdaccio registry and store artifacts in a new temporary folder on the os - * @returns {Promise<{ url: URL; storage: string; }>} - */ -export const startRegistry = async () => { - const config = getVerdaccioConfig() - - // Remove netlify-cli from the verdaccio storage because we are going to publish it in a second - await rm(join(config.storage, 'netlify-cli'), { force: true, recursive: true }) - - // generate a random starting port to avoid race condition inside the promise when running a large - // number in parallel - const startPort = Math.floor(Math.random() * END_PORT_RANGE) + START_PORT_RANGE - const freePort = await getPort({ host: 'localhost', port: startPort }) - - return await pTimeout(runVerdaccio(config, freePort), { - milliseconds: VERDACCIO_TIMEOUT_MILLISECONDS, - fallback: 'Starting Verdaccio timed out', - }) -} - -/** - * Setups the environment with publishing the CLI to a intermediate registry - * and creating a folder to start with. - * @returns { - * registry: string, - * workspace: string, - * cleanup: () => Promise - * } - */ -export const setup = async () => { - const { storage, url } = await startRegistry() - const workspace = await mkdtemp(`${tmpdir()}${sep}e2e-test-`) - - const npmrc = join(rootDir, '.npmrc') - const registryWithAuth = `//${url.hostname}:${url.port}/:_authToken=dummy` - let backupNpmrc - - /** Cleans up everything */ - const cleanup = async () => { - // remote temp folders - await rm(workspace, { force: true, recursive: true }) - } - - env.npm_config_registry = url - - try { - if (await fileExistsAsync(npmrc)) { - backupNpmrc = await readFile(npmrc, 'utf-8') - await appendFile(npmrc, registryWithAuth) - } else { - await writeFile(npmrc, registryWithAuth, 'utf-8') - } - - // publish the CLI package to our registry - await execa('npm', ['publish', `--registry=${url}`, '--tag=testing'], { - stdio: env.DEBUG ? 'inherit' : 'ignore', - cwd: rootDir, - }) - - await execa('npm', ['install', '--no-audit'], { cwd: rootDir }) - - console.log(`------------------------------------------ - Published to ${url} - Verdaccio: ${storage} - Workspace: ${workspace} -------------------------------------------`) - } catch (error_) { - await cleanup() - throw new Error( - `npm publish failed for registry ${url.href} - -${error_ instanceof Error ? error_.message : error_}`, - ) - } finally { - // restore .npmrc - // eslint-disable-next-line unicorn/prefer-ternary - if (backupNpmrc) { - await writeFile(npmrc, backupNpmrc) - } else { - await rm(npmrc, { force: true, recursive: true }) - } - } - - return { - registry: url, - workspace, - cleanup, - } -} diff --git a/vitest.e2e.config.ts b/vitest.e2e.config.ts index 20d0ed23457..122e91f01dc 100644 --- a/vitest.e2e.config.ts +++ b/vitest.e2e.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { - include: ['e2e/**/*.e2e.js'], + include: ['e2e/**/*.e2e.[jt]s'], testTimeout: 600_000, deps: { external: ['**/fixtures/**', '**/node_modules/**'], From 82f3522d99effc848c2417f3a866e0032aae5c22 Mon Sep 17 00:00:00 2001 From: Nathan Houle Date: Tue, 25 Mar 2025 14:13:15 -0700 Subject: [PATCH 2/2] test(e2e): always test against npm+pnpm+yarn --- .github/workflows/e2e-tests.yml | 12 +++----- e2e/install.e2e.ts | 50 +++++++++++++-------------------- 2 files changed, 23 insertions(+), 39 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e0126a569b4..6c3caae6a73 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -35,11 +35,9 @@ jobs: cache: npm check-latest: true - - name: Cache verdaccio storage - uses: actions/cache@v4 + - uses: pnpm/action-setup@v4 with: - path: ./.verdaccio-storage - key: verdaccio-e2e-cli-${{ hashFiles('./package-lock.json') }} + version: 10 - name: Install dependencies run: npm ci --no-audit @@ -91,11 +89,9 @@ jobs: cache: npm check-latest: true - - name: Cache verdaccio storage - uses: actions/cache@v4 + - uses: pnpm/action-setup@v4 with: - path: ./.verdaccio-storage - key: verdaccio-e2e-cli-${{ hashFiles('./package-lock.json') }} + version: 10 - name: Install dependencies run: npm ci --no-audit diff --git a/e2e/install.e2e.ts b/e2e/install.e2e.ts index 2506bd549f6..01df5bac621 100644 --- a/e2e/install.e2e.ts +++ b/e2e/install.e2e.ts @@ -136,15 +136,6 @@ const itWithMockNpmRegistry = it.extend<{ registry: { address: string; cwd: stri }, }) -const doesPackageManagerExist = (packageManager: string): boolean => { - try { - execSync(`${packageManager} --version`) - return true - } catch { - return false - } -} - const tests: [packageManager: string, config: { install: [cmd: string, args: string[]]; lockfile: string }][] = [ [ 'npm', @@ -170,29 +161,26 @@ const tests: [packageManager: string, config: { install: [cmd: string, args: str ] describe.each(tests)('%s → installs the cli and runs the help command without error', (packageManager, config) => { - itWithMockNpmRegistry.runIf(doesPackageManagerExist(packageManager))( - 'installs the cli and runs the help command without error', - async ({ registry }) => { - const cwd = registry.cwd - await execa(...config.install, { - cwd, - env: { npm_config_registry: registry.address }, - stdio: debug.enabled ? 'inherit' : 'ignore', - }) + itWithMockNpmRegistry('installs the cli and runs the help command without error', async ({ registry }) => { + const cwd = registry.cwd + await execa(...config.install, { + cwd, + env: { npm_config_registry: registry.address }, + stdio: debug.enabled ? 'inherit' : 'ignore', + }) - expect( - existsSync(path.join(cwd, config.lockfile)), - `Generated lock file ${config.lockfile} does not exist in ${cwd}`, - ).toBe(true) + expect( + existsSync(path.join(cwd, config.lockfile)), + `Generated lock file ${config.lockfile} does not exist in ${cwd}`, + ).toBe(true) - const binary = path.resolve(path.join(cwd, `./node_modules/.bin/netlify${platform() === 'win32' ? '.cmd' : ''}`)) - const { stdout } = await execa(binary, ['help'], { cwd }) + const binary = path.resolve(path.join(cwd, `./node_modules/.bin/netlify${platform() === 'win32' ? '.cmd' : ''}`)) + const { stdout } = await execa(binary, ['help'], { cwd }) - expect(stdout.trim(), `Help command does not start with 'VERSION':\n\n${stdout}`).toMatch(/^VERSION/) - expect(stdout, `Help command does not include 'netlify-cli/${pkg.version}':\n\n${stdout}`).toContain( - `netlify-cli/${pkg.version}`, - ) - expect(stdout, `Help command does not include '$ netlify [COMMAND]':\n\n${stdout}`).toMatch('$ netlify [COMMAND]') - }, - ) + expect(stdout.trim(), `Help command does not start with 'VERSION':\n\n${stdout}`).toMatch(/^VERSION/) + expect(stdout, `Help command does not include 'netlify-cli/${pkg.version}':\n\n${stdout}`).toContain( + `netlify-cli/${pkg.version}`, + ) + expect(stdout, `Help command does not include '$ netlify [COMMAND]':\n\n${stdout}`).toMatch('$ netlify [COMMAND]') + }) })