diff --git a/.gitignore b/.gitignore index f8e97ed5f..215656b70 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ tsconfig.schemastore-schema.json /website/static/api /tsconfig.tsbuildinfo /temp +/.tmp diff --git a/development-docs/caches.md b/development-docs/caches.md new file mode 100644 index 000000000..ec6b3e8cc --- /dev/null +++ b/development-docs/caches.md @@ -0,0 +1,32 @@ +Lots of caching in ts-node. + +## Caches + +### FS cache: +caches results of primitive ts.sys.readFile, etc operations +Shared across compiler and config loader + +### fileContents (and fileVersions) cache: +sits in front of fs cache. +Node.js module loading mechanism reads file contents from disk. That's put into this cache. + +### Output cache: +Caches the emitted JS syntax from compilation. +Has appended //# sourcemap comments. +source-map-support reads from here before fallback to filesystem. + +### source-map-support cache: +caches fs.readFile calls +overlayed by output cache above +overlayed by sourcesContents from parsed sourcemaps + +### SourceFile cache: (does not exist today) +for Compiler API codepath +to avoid re-parsing SourceFile repeatedly + +## Questions + +If both: +- source-map-support caches a sourcesContents string of a .ts file +- cachedFsReader caches the same .ts file from disk +...which is used by source-map-support? Does it matter since they should be identical? diff --git a/development-docs/ts-sys.ts b/development-docs/ts-sys.ts new file mode 100644 index 000000000..747b11102 --- /dev/null +++ b/development-docs/ts-sys.ts @@ -0,0 +1,36 @@ +import ts = require("typescript"); +import { getPatternFromSpec } from "../src/ts-internals"; + +// Notes and demos to understand `ts.sys` + +console.dir(ts.sys.getCurrentDirectory()); +// Gets names (not paths) of all directories that are direct children of given path +// Never throws +// Accepts trailing `/` or not +console.dir(ts.sys.getDirectories(ts.sys.getCurrentDirectory())); + +///// + +// Returns array of absolute paths +// Never returns directories; only files + +// Values can have period or not; are interpreted as a suffix ('o.svg' matches logo.svg; seems to also match if you embed / directory delimiters) +// [''] is the same as undefined; returns everything +const extensions: string[] | undefined = ['']; +// Supports wildcards; ts-style globs? +const exclude: string[] | undefined = undefined; +const include: string[] | undefined = ['*/????????????']; +// Depth == 0 is the same as undefined: unlimited depth +// Depth == 1 is only direct children of directory +const depth: number | undefined = undefined; +console.dir(ts.sys.readDirectory(ts.sys.getCurrentDirectory(), extensions, exclude, include, depth)); + +// To overlay virtual filesystem contents over `ts.sys.readDirectory`, try this: +// start with array of all virtual files +// Filter by those having base directory prefix +// if extensions is array, do an `endsWith` filter +// if exclude is an array, use `getPatternFromSpec` and filter out anything that matches +// if include is an array, use `getPatternFromSpec` and filter out anything that does not match at least one +// if depth is non-zero, count the number of directory delimiters following the base directory prefix + +console.log(getPatternFromSpec('foo/*/bar', ts.sys.getCurrentDirectory())); diff --git a/dist-raw/node-cjs-helpers.d.ts b/dist-raw/node-cjs-helpers.d.ts deleted file mode 100644 index a57c2f831..000000000 --- a/dist-raw/node-cjs-helpers.d.ts +++ /dev/null @@ -1 +0,0 @@ -export function addBuiltinLibsToObject(object: any): void; diff --git a/dist-raw/node-cjs-loader-utils.js b/dist-raw/node-cjs-loader-utils.js index b7ec0d531..d2a1aabf9 100644 --- a/dist-raw/node-cjs-loader-utils.js +++ b/dist-raw/node-cjs-loader-utils.js @@ -3,131 +3,136 @@ // Each function and variable below must have a comment linking to the source in node's github repo. const path = require('path'); -const packageJsonReader = require('./node-package-json-reader'); const {JSONParse} = require('./node-primordials'); const {normalizeSlashes} = require('../dist/util'); -module.exports.assertScriptCanLoadAsCJSImpl = assertScriptCanLoadAsCJSImpl; +/** @param {ReturnType} packageJsonReader */ +function createNodeCjsLoaderUtils(packageJsonReader) { -/** - * copied from Module._extensions['.js'] - * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120 - * @param {import('../src/index').Service} service - * @param {NodeJS.Module} module - * @param {string} filename - */ -function assertScriptCanLoadAsCJSImpl(service, module, filename) { - const pkg = readPackageScope(filename); + /** + * copied from Module._extensions['.js'] + * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120 + * @param {import('../src/index').Service} service + * @param {NodeJS.Module} module + * @param {string} filename + */ + function assertScriptCanLoadAsCJSImpl(service, module, filename) { + const pkg = readPackageScope(filename); - // ts-node modification: allow our configuration to override - const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename)); - if(tsNodeClassification.moduleType === 'cjs') return; + // ts-node modification: allow our configuration to override + const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename)); + if(tsNodeClassification.moduleType === 'cjs') return; - // Function require shouldn't be used in ES modules. - if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) { - const parentPath = module.parent && module.parent.filename; - const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null; - throw createErrRequireEsm(filename, parentPath, packageJsonPath); + // Function require shouldn't be used in ES modules. + if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) { + const parentPath = module.parent && module.parent.filename; + const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null; + throw createErrRequireEsm(filename, parentPath, packageJsonPath); + } } -} -// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301 -function readPackageScope(checkPath) { - const rootSeparatorIndex = checkPath.indexOf(path.sep); - let separatorIndex; - while ( - (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex - ) { - checkPath = checkPath.slice(0, separatorIndex); - if (checkPath.endsWith(path.sep + 'node_modules')) - return false; - const pjson = readPackage(checkPath); - if (pjson) return { - path: checkPath, - data: pjson - }; + // Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301 + function readPackageScope(checkPath) { + const rootSeparatorIndex = checkPath.indexOf(path.sep); + let separatorIndex; + while ( + (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex + ) { + checkPath = checkPath.slice(0, separatorIndex); + if (checkPath.endsWith(path.sep + 'node_modules')) + return false; + const pjson = readPackage(checkPath); + if (pjson) return { + path: checkPath, + data: pjson + }; + } + return false; } - return false; -} -// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249 -const packageJsonCache = new Map(); + // Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249 + const packageJsonCache = new Map(); -// Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304 -function readPackage(requestPath) { - const jsonPath = path.resolve(requestPath, 'package.json'); + // Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304 + function readPackage(requestPath) { + const jsonPath = path.resolve(requestPath, 'package.json'); - const existing = packageJsonCache.get(jsonPath); - if (existing !== undefined) return existing; + const existing = packageJsonCache.get(jsonPath); + if (existing !== undefined) return existing; - const result = packageJsonReader.read(jsonPath); - const json = result.containsKeys === false ? '{}' : result.string; - if (json === undefined) { - packageJsonCache.set(jsonPath, false); - return false; - } + const result = packageJsonReader.read(jsonPath); + const json = result.containsKeys === false ? '{}' : result.string; + if (json === undefined) { + packageJsonCache.set(jsonPath, false); + return false; + } - try { - const parsed = JSONParse(json); - const filtered = { - name: parsed.name, - main: parsed.main, - exports: parsed.exports, - imports: parsed.imports, - type: parsed.type - }; - packageJsonCache.set(jsonPath, filtered); - return filtered; - } catch (e) { - e.path = jsonPath; - e.message = 'Error parsing ' + jsonPath + ': ' + e.message; - throw e; + try { + const parsed = JSONParse(json); + const filtered = { + name: parsed.name, + main: parsed.main, + exports: parsed.exports, + imports: parsed.imports, + type: parsed.type + }; + packageJsonCache.set(jsonPath, filtered); + return filtered; + } catch (e) { + e.path = jsonPath; + e.message = 'Error parsing ' + jsonPath + ': ' + e.message; + throw e; + } } -} -// Native ERR_REQUIRE_ESM Error is declared here: -// https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313 -// Error class factory is implemented here: -// function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341 -// function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278 -// The code below should create an error that matches the native error as closely as possible. -// Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error. -function createErrRequireEsm(filename, parentPath, packageJsonPath) { - const code = 'ERR_REQUIRE_ESM' - const err = new Error(getMessage(filename, parentPath, packageJsonPath)) - // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field. - // This trick is copied from node's source. - err.name = `Error [${ code }]` - err.stack - Object.defineProperty(err, 'name', { - value: 'Error', - enumerable: false, - writable: true, - configurable: true - }) - err.code = code - return err + // Native ERR_REQUIRE_ESM Error is declared here: + // https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313 + // Error class factory is implemented here: + // function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341 + // function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278 + // The code below should create an error that matches the native error as closely as possible. + // Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error. + function createErrRequireEsm(filename, parentPath, packageJsonPath) { + const code = 'ERR_REQUIRE_ESM' + const err = new Error(getMessage(filename, parentPath, packageJsonPath)) + // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field. + // This trick is copied from node's source. + err.name = `Error [${ code }]` + err.stack + Object.defineProperty(err, 'name', { + value: 'Error', + enumerable: false, + writable: true, + configurable: true + }) + err.code = code + return err - // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311 - // so that our error message is identical to the native message. - function getMessage(filename, parentPath = null, packageJsonPath = null) { - const ext = path.extname(filename) - let msg = `Must use import to load ES Module: ${filename}`; - if (parentPath && packageJsonPath) { - const path = require('path'); - const basename = path.basename(filename) === path.basename(parentPath) ? - filename : path.basename(filename); - msg += - '\nrequire() of ES modules is not supported.\nrequire() of ' + - `${filename} ${parentPath ? `from ${parentPath} ` : ''}` + - `is an ES module file as it is a ${ext} file whose nearest parent ` + - `package.json contains "type": "module" which defines all ${ext} ` + - 'files in that package scope as ES modules.\nInstead ' + - 'change the requiring code to use ' + - 'import(), or remove "type": "module" from ' + - `${packageJsonPath}.\n`; + // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311 + // so that our error message is identical to the native message. + function getMessage(filename, parentPath = null, packageJsonPath = null) { + const ext = path.extname(filename) + let msg = `Must use import to load ES Module: ${filename}`; + if (parentPath && packageJsonPath) { + const path = require('path'); + const basename = path.basename(filename) === path.basename(parentPath) ? + filename : path.basename(filename); + msg += + '\nrequire() of ES modules is not supported.\nrequire() of ' + + `${filename} ${parentPath ? `from ${parentPath} ` : ''}` + + `is an ES module file as it is a ${ext} file whose nearest parent ` + + `package.json contains "type": "module" which defines all ${ext} ` + + 'files in that package scope as ES modules.\nInstead ' + + 'change the requiring code to use ' + + 'import(), or remove "type": "module" from ' + + `${packageJsonPath}.\n`; + return msg; + } return msg; } - return msg; } + + return {assertScriptCanLoadAsCJSImpl}; } + +module.exports.createNodeCjsLoaderUtils = createNodeCjsLoaderUtils; diff --git a/dist-raw/node-esm-default-get-format.js b/dist-raw/node-esm-default-get-format.js index d8af956f3..5c2fec59c 100644 --- a/dist-raw/node-esm-default-get-format.js +++ b/dist-raw/node-esm-default-get-format.js @@ -15,7 +15,6 @@ const experimentalJsonModules = getOptionValue('--experimental-json-modules'); const experimentalSpeciferResolution = getOptionValue('--experimental-specifier-resolution'); const experimentalWasmModules = getOptionValue('--experimental-wasm-modules'); -const { getPackageType } = require('./node-esm-resolve-implementation.js').createResolve({tsExtensions: [], jsExtensions: []}); const { URL, fileURLToPath } = require('url'); const { ERR_UNKNOWN_FILE_EXTENSION } = require('./node-errors').codes; @@ -41,6 +40,9 @@ if (experimentalWasmModules) if (experimentalJsonModules) extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json'; +function createDefaultGetFormat(getPackageType) { + +// Intentionally unindented to simplify the diff function defaultGetFormat(url, context, defaultGetFormatUnused) { if (StringPrototypeStartsWith(url, 'node:')) { return { format: 'builtin' }; @@ -80,4 +82,9 @@ function defaultGetFormat(url, context, defaultGetFormatUnused) { } return { format: null }; } -exports.defaultGetFormat = defaultGetFormat; +return { + defaultGetFormat: /** @type {import('../src/esm').GetFormatHook} */(defaultGetFormat) +}; +} + +exports.createDefaultGetFormat = createDefaultGetFormat; diff --git a/dist-raw/node-esm-resolve-implementation.js b/dist-raw/node-esm-resolve-implementation.js index 04ea84668..a0f5c40b6 100644 --- a/dist-raw/node-esm-resolve-implementation.js +++ b/dist-raw/node-esm-resolve-implementation.js @@ -95,16 +95,26 @@ const { const CJSModule = Module; // const packageJsonReader = require('internal/modules/package_json_reader'); -const packageJsonReader = require('./node-package-json-reader'); + +const { createDefaultGetFormat } = require('./node-esm-default-get-format'); + const userConditions = getOptionValue('--conditions'); const DEFAULT_CONDITIONS = ObjectFreeze(['node', 'import', ...userConditions]); const DEFAULT_CONDITIONS_SET = new SafeSet(DEFAULT_CONDITIONS); const pendingDeprecation = getOptionValue('--pending-deprecation'); +/** + * @param {{ + * tsExtensions: string[]; + * jsExtensions: string[]; + * preferTsExts: boolean | undefined; + * nodePackageJsonReader: ReturnType; + * }} opts + */ function createResolve(opts) { -// TODO receive cached fs implementations here -const {tsExtensions, jsExtensions, preferTsExts} = opts; +const {tsExtensions, jsExtensions, preferTsExts, nodePackageJsonReader: packageJsonReader} = opts; +const {defaultGetFormat} = createDefaultGetFormat(getPackageType); const emittedPackageWarnings = new SafeSet(); function emitFolderMapDeprecation(match, pjsonUrl, isExports, base) { @@ -971,9 +981,8 @@ return { encodedSepRegEx, getPackageType, packageExportsResolve, - packageImportsResolve + packageImportsResolve, + defaultGetFormat }; } -module.exports = { - createResolve -}; +module.exports.createResolve = createResolve; diff --git a/dist-raw/node-internal-fs.js b/dist-raw/node-internal-fs.js index d9a2528dd..e649dfc27 100644 --- a/dist-raw/node-internal-fs.js +++ b/dist-raw/node-internal-fs.js @@ -1,22 +1,24 @@ const fs = require('fs'); -// In node's core, this is implemented in C -// https://github.com/nodejs/node/blob/v15.3.0/src/node_file.cc#L891-L985 -function internalModuleReadJSON(path) { - let string - try { - string = fs.readFileSync(path, 'utf8') - } catch (e) { - if (e.code === 'ENOENT') return [] - throw e +function createNodeInternalModuleReadJSON() { + // In node's core, this is implemented in C + // https://github.com/nodejs/node/blob/v15.3.0/src/node_file.cc#L891-L985 + function internalModuleReadJSON(path) { + let string + try { + string = fs.readFileSync(path, 'utf8') + } catch (e) { + if (e.code === 'ENOENT') return [] + throw e + } + // Node's implementation checks for the presence of relevant keys: main, name, type, exports, imports + // Node does this for performance to skip unnecessary parsing. + // This would slow us down and, based on our usage, we can skip it. + const containsKeys = true + return [string, containsKeys] } - // Node's implementation checks for the presence of relevant keys: main, name, type, exports, imports - // Node does this for performance to skip unnecessary parsing. - // This would slow us down and, based on our usage, we can skip it. - const containsKeys = true - return [string, containsKeys] + + return internalModuleReadJSON; } -module.exports = { - internalModuleReadJSON -}; +module.exports.createNodeInternalModuleReadJSON = createNodeInternalModuleReadJSON; diff --git a/dist-raw/node-package-json-reader.js b/dist-raw/node-package-json-reader.js index e9f82c6f4..179be01ba 100644 --- a/dist-raw/node-package-json-reader.js +++ b/dist-raw/node-package-json-reader.js @@ -2,11 +2,13 @@ 'use strict'; const { SafeMap } = require('./node-primordials'); -const { internalModuleReadJSON } = require('./node-internal-fs'); const { pathToFileURL } = require('url'); const { toNamespacedPath } = require('path'); // const { getOptionValue } = require('./node-options'); +/** @param {ReturnType} internalModuleReadJSON */ +function createNodePackageJsonReader(internalModuleReadJSON) { +// Intentionally un-indented to keep diff small without needing to mess with whitespace-ignoring flags const cache = new SafeMap(); let manifest; @@ -41,4 +43,7 @@ function read(jsonPath) { return result; } -module.exports = { read }; +return { read }; +} + +module.exports.createNodePackageJsonReader = createNodePackageJsonReader; diff --git a/package.json b/package.json index 060a78c44..dd20399da 100644 --- a/package.json +++ b/package.json @@ -56,11 +56,11 @@ "scripts": { "lint": "prettier --check .", "lint-fix": "prettier --write .", - "clean": "rimraf dist tsconfig.schema.json tsconfig.schemastore-schema.json tsconfig.tsbuildinfo tests/ts-node-packed.tgz", + "clean": "rimraf dist tsconfig.schema.json tsconfig.schemastore-schema.json tests/ts-node-packed.tgz .tmp", "rebuild": "npm run clean && npm run build", "build": "npm run build-nopack && npm run build-pack", "build-nopack": "npm run build-tsc && npm run build-configSchema", - "build-tsc": "tsc", + "build-tsc": "tsc --build ./tsconfig.declarations.json && rimraf .tmp/declarations/src && tsc --build ./tsconfig.build.json", "build-configSchema": "typescript-json-schema --topRef --refs --validationKeywords allOf --out tsconfig.schema.json tsconfig.build-schema.json TsConfigSchema && node --require ./register ./scripts/create-merged-schema", "build-pack": "node ./scripts/build-pack.js", "test-spec": "ava", diff --git a/src/configuration.ts b/src/configuration.ts index a970b49c4..0a7b5764f 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -1,5 +1,6 @@ import { resolve, dirname } from 'path'; import type * as _ts from 'typescript'; +import type { TsSysFsReader } from './fs'; import { CreateOptions, DEFAULTS, @@ -61,6 +62,7 @@ function fixConfig(ts: TSCommon, config: _ts.ParsedCommandLine) { export function readConfig( cwd: string, ts: TSCommon, + fsReader: TsSysFsReader, rawApiOptions: CreateOptions ): { /** @@ -90,8 +92,6 @@ export function readConfig( const projectSearchDir = resolve(cwd, rawApiOptions.projectSearchDir ?? cwd); const { - fileExists = ts.sys.fileExists, - readFile = ts.sys.readFile, skipProject = DEFAULTS.skipProject, project = DEFAULTS.project, } = rawApiOptions; @@ -100,7 +100,7 @@ export function readConfig( if (!skipProject) { configFilePath = project ? resolve(cwd, project) - : ts.findConfigFile(projectSearchDir, fileExists); + : ts.findConfigFile(projectSearchDir, fsReader.fileExists); if (configFilePath) { let pathToNextConfigInChain = configFilePath; @@ -109,7 +109,10 @@ export function readConfig( // Follow chain of "extends" while (true) { - const result = ts.readConfigFile(pathToNextConfigInChain, readFile); + const result = ts.readConfigFile( + pathToNextConfigInChain, + fsReader.readFile + ); // Return diagnostics. if (result.error) { @@ -133,10 +136,10 @@ export function readConfig( const resolvedExtendedConfigPath = tsInternals.getExtendsConfigPath( c.extends, { - fileExists, - readDirectory: ts.sys.readDirectory, - readFile, - useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, + fileExists: fsReader.fileExists, + readDirectory: fsReader.readDirectory, + readFile: fsReader.readFile, + useCaseSensitiveFileNames: fsReader.useCaseSensitiveFileNames, trace, }, bp, @@ -226,10 +229,10 @@ export function readConfig( ts.parseJsonConfigFileContent( config, { - fileExists, - readFile, - readDirectory: ts.sys.readDirectory, - useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, + fileExists: fsReader.fileExists, + readFile: fsReader.readFile, + readDirectory: fsReader.readDirectory, + useCaseSensitiveFileNames: fsReader.useCaseSensitiveFileNames, }, basePath, undefined, diff --git a/src/diagnostics.ts b/src/diagnostics.ts new file mode 100644 index 000000000..bee56aad5 --- /dev/null +++ b/src/diagnostics.ts @@ -0,0 +1,21 @@ +import { env, yn } from './util'; + +/** + * Debugging `ts-node`. + */ +const shouldDebug = yn(env.TS_NODE_DEBUG); +/** @internal */ +export const debug = shouldDebug + ? (...args: any) => + console.log(`[ts-node ${new Date().toISOString()}]`, ...args) + : () => undefined; +/** @internal */ +export const debugFn = shouldDebug + ? (key: string, fn: (arg: T) => U) => { + let i = 0; + return (x: T) => { + debug(key, x, ++i); + return fn(x); + }; + } + : (_: string, fn: (arg: T) => U) => fn; diff --git a/src/esm.ts b/src/esm.ts index c83fd22c4..568c42073 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -15,10 +15,7 @@ import { import { extname } from 'path'; import * as assert from 'assert'; import { normalizeSlashes } from './util'; -const { - createResolve, -} = require('../dist-raw/node-esm-resolve-implementation'); -const { defaultGetFormat } = require('../dist-raw/node-esm-default-get-format'); +import { createResolve } from '../dist-raw/node-esm-resolve-implementation'; // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts @@ -83,6 +80,11 @@ export type NodeLoaderHooksFormat = | 'module' | 'wasm'; +/** @internal */ +export type GetFormatHook = NonNullable< + ReturnType['getFormat'] +>; + /** @internal */ export function registerAndCreateEsmHooks(opts?: RegisterOptions) { // Automatically performs registration just like `-r ts-node/register` @@ -98,6 +100,7 @@ export function createEsmHooks(tsNodeService: Service) { const nodeResolveImplementation = createResolve({ ...getExtensions(tsNodeService.config), preferTsExts: tsNodeService.options.preferTsExts, + nodePackageJsonReader: tsNodeService.nodePackageJsonReader, }); // The hooks API changed in node version X so we need to check for backwards compatibility. @@ -169,7 +172,13 @@ export function createEsmHooks(tsNodeService: Service) { // otherwise call the old getFormat() hook using node's old built-in defaultGetFormat() that ships with ts-node const format = context.format ?? - (await getFormat(url, context, defaultGetFormat)).format; + ( + await getFormat( + url, + context, + nodeResolveImplementation.defaultGetFormat + ) + ).format; let source = undefined; if (format !== 'builtin' && format !== 'commonjs') { diff --git a/src/fs.ts b/src/fs.ts new file mode 100644 index 000000000..608cd0456 --- /dev/null +++ b/src/fs.ts @@ -0,0 +1,110 @@ +import type * as _ts from 'typescript'; +import * as fs from 'fs'; +import { debugFn } from './diagnostics'; +import { cachedLookup } from './util'; + +// Types of fs implementation: +// +// Cached fs +// Proxies to real filesystem, caches results in-memory. +// Has invalidation APIs to support `delete require.cache[foo]` +// +// Overlay fs +// Is writable. Written contents remain in memory. +// Written contents can be serialized / deserialized. +// Read calls return from in-memory, proxy to another FS if not found. + +/* + +require('./dist/bar.js') +dist/bar.js exists +preferTsExts=true we should resolve to ./src/bar.ts +preferTsExts=false we should resolve to ./dist/bar.js + - read file from the filesystem? + - read file from the overlay fs? + +*/ + +/** @internal */ +export type NodeFsReader = { + readFileSync(path: string, encoding: 'utf8'): string; + statSync(path: string): fs.Stats; +}; + +/** + * @internal + * Since `useCaseSensitiveFileNames` is required to know how to cache, we expose it on the interface + */ +export type TsSysFsReader = Pick< + _ts.System, + | 'directoryExists' + | 'fileExists' + | 'getDirectories' + | 'readDirectory' + | 'readFile' + | 'realpath' + | 'resolvePath' + | 'useCaseSensitiveFileNames' +>; +/** since I've never hit code that needs these functions implemented */ +type TsSysFullFsReader = TsSysFsReader & + Pick<_ts.System, 'getFileSize' | 'getModifiedTime'>; +type TsSysFsWriter = Pick< + _ts.System, + 'createDirectory' | 'deleteFile' | 'setModifiedTime' | 'writeFile' +>; +type TsSysFsWatcher = Pick<_ts.System, 'watchDirectory' | 'watchFile'>; + +/** @internal */ +export type CachedFsReader = ReturnType; + +// Start with no caching; then add it bit by bit +/** @internal */ +export function createCachedFsReader(reader: TsSysFsReader) { + // TODO if useCaseSensitive is false, then lowercase all cache keys? + + const fileContentsCache = new Map(); + const normalizeFileCacheKey = reader.useCaseSensitiveFileNames + ? (key: string) => key + : (key: string) => key.toLowerCase(); + + function invalidateFileContents() {} + function invalidateFileExistence() {} + + const sys: TsSysFsReader = { + ...reader, + directoryExists: cachedLookup( + debugFn('directoryExists', reader.directoryExists) + ), + fileExists: cachedLookup(debugFn('fileExists', reader.fileExists)), + getDirectories: cachedLookup( + debugFn('getDirectories', reader.getDirectories) + ), + readFile: cachedLookup(debugFn('readFile', reader.readFile)), + realpath: reader.realpath + ? cachedLookup(debugFn('realpath', reader.realpath)) + : undefined, + resolvePath: cachedLookup(debugFn('resolvePath', reader.resolvePath)), + }; + const enoentError = new Error() as Error & { code: 'ENOENT' }; + enoentError.code = 'ENOENT'; + const nodeFs: NodeFsReader = { + readFileSync(path: string, encoding: 'utf8') { + // TODO It is unnecessarily messy to implement node's `readFileSync` on top of TS's `readFile`. Refactor this. + const ret = sys.readFile(path); + if (typeof ret !== 'string') { + throw enoentError; + } + return ret; + }, + statSync: cachedLookup( + debugFn('statSync', fs.statSync) + ) as NodeFsReader['statSync'], + }; + return { + sys, + nodeFs, + invalidateFileContents, + invalidateFileExistence, + }; +} diff --git a/src/index.ts b/src/index.ts index 977922ea5..0d6db0d0a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,14 +8,7 @@ import { BaseError } from 'make-error'; import type * as _ts from 'typescript'; import type { Transpiler, TranspilerFactory } from './transpilers/types'; -import { - assign, - cachedLookup, - normalizeSlashes, - parse, - split, - yn, -} from './util'; +import { assign, env, normalizeSlashes, parse, split, yn } from './util'; import { readConfig } from './configuration'; import type { TSCommon, TSInternal } from './ts-compiler-types'; import { @@ -24,6 +17,11 @@ import { } from './module-type-classifier'; import { createResolverFunctions } from './resolver-functions'; import type { createEsmHooks as createEsmHooksFn } from './esm'; +import { CachedFsReader, createCachedFsReader } from './fs'; +import { debug } from './diagnostics'; +import type { createNodeCjsLoaderUtils as _createNodeCjsLoaderUtils } from '../dist-raw/node-cjs-loader-utils'; +import { createNodePackageJsonReader } from '../dist-raw/node-package-json-reader'; +import { createNodeInternalModuleReadJSON as createNodeInternalModuleReadJSON } from '../dist-raw/node-internal-fs'; export { TSCommon }; export { @@ -50,9 +48,15 @@ export type { * Does this version of node obey the package.json "type" field * and throw ERR_REQUIRE_ESM when attempting to require() an ESM modules. */ +// TODO remove this; today we only support node >=12 const engineSupportsPackageTypeField = parseInt(process.versions.node.split('.')[0], 10) >= 12; +const createNodeCjsLoaderUtils = engineSupportsPackageTypeField + ? (require('../dist-raw/node-cjs-loader-utils') + .createNodeCjsLoaderUtils as typeof _createNodeCjsLoaderUtils) + : undefined; + /** @internal */ export function versionGteLt( version: string, @@ -80,22 +84,6 @@ export function versionGteLt( } } -/** - * Assert that script can be loaded as CommonJS when we attempt to require it. - * If it should be loaded as ESM, throw ERR_REQUIRE_ESM like node does. - * - * Loaded conditionally so we don't need to support older node versions - */ -let assertScriptCanLoadAsCJS: ( - service: Service, - module: NodeJS.Module, - filename: string -) => void = engineSupportsPackageTypeField - ? require('../dist-raw/node-cjs-loader-utils').assertScriptCanLoadAsCJSImpl - : () => { - /* noop */ - }; - /** * Registered `ts-node` instance information. */ @@ -112,8 +100,6 @@ declare global { } } -/** @internal */ -export const env = process.env as ProcessEnv; /** * Declare all env vars, to aid discoverability. * If an env var affects ts-node's behavior, it should not be buried somewhere in our codebase. @@ -152,25 +138,6 @@ export interface ProcessEnv { */ export const INSPECT_CUSTOM = util.inspect.custom || 'inspect'; -/** - * Debugging `ts-node`. - */ -const shouldDebug = yn(env.TS_NODE_DEBUG); -/** @internal */ -export const debug = shouldDebug - ? (...args: any) => - console.log(`[ts-node ${new Date().toISOString()}]`, ...args) - : () => undefined; -const debugFn = shouldDebug - ? (key: string, fn: (arg: T) => U) => { - let i = 0; - return (x: T) => { - debug(key, x, ++i); - return fn(x); - }; - } - : (_: string, fn: (arg: T) => U) => fn; - /** * Export the current version. */ @@ -465,6 +432,22 @@ export interface Service { installSourceMapSupport(): void; /** @internal */ enableExperimentalEsmLoaderInterop(): void; + /** @internal */ + cachedFsReader: CachedFsReader; + /** @internal */ + nodeInternalModuleReadJson: ReturnType< + typeof import('../dist-raw/node-internal-fs').createNodeInternalModuleReadJSON + >; + /** @internal */ + nodePackageJsonReader: ReturnType< + typeof import('../dist-raw/node-package-json-reader').createNodePackageJsonReader + >; + /** @internal */ + nodeCjsLoaderUtils: + | ReturnType< + typeof import('../dist-raw/node-cjs-loader-utils').createNodeCjsLoaderUtils + > + | undefined; } /** @@ -562,13 +545,29 @@ export function create(rawOptions: CreateOptions = {}): Service { rawOptions.projectSearchDir ?? rawOptions.project ?? cwd ); + const cachedFsReader = createCachedFsReader({ + directoryExists: ts.sys.directoryExists, + fileExists: rawOptions.fileExists ?? ts.sys.fileExists, + getDirectories: ts.sys.getDirectories, + readDirectory: ts.sys.readDirectory, + readFile: rawOptions.readFile ?? ts.sys.readFile, + resolvePath: ts.sys.resolvePath, + realpath: ts.sys.realpath, + useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, + }); + const nodeInternalModuleReadJson = createNodeInternalModuleReadJSON(); + const nodePackageJsonReader = createNodePackageJsonReader( + nodeInternalModuleReadJson + ); + const nodeCjsLoaderUtils = createNodeCjsLoaderUtils?.(nodePackageJsonReader); + // Read config file and merge new options between env and CLI options. const { configFilePath, config, tsNodeOptionsFromTsconfig, optionBasePaths, - } = readConfig(cwd, ts, rawOptions); + } = readConfig(cwd, ts, cachedFsReader.sys, rawOptions); const options = assign( {}, DEFAULTS, @@ -608,8 +607,6 @@ export function create(rawOptions: CreateOptions = {}): Service { ({ compiler, ts } = loadCompiler(options.compiler, configFilePath)); } - const readFile = options.readFile || ts.sys.readFile; - const fileExists = options.fileExists || ts.sys.fileExists; // typeCheck can override transpileOnly, useful for CLI flag to override config file const transpileOnly = options.transpileOnly === true && options.typeCheck !== true; @@ -657,7 +654,8 @@ export function create(rawOptions: CreateOptions = {}): Service { const diagnosticHost: _ts.FormatDiagnosticsHost = { getNewLine: () => ts.sys.newLine, getCurrentDirectory: () => cwd, - getCanonicalFileName: ts.sys.useCaseSensitiveFileNames + // TODO replace with `ts.createGetCanonicalFileName`? + getCanonicalFileName: cachedFsReader.sys.useCaseSensitiveFileNames ? (x) => x : (x) => x.toLowerCase(), }; @@ -718,7 +716,11 @@ export function create(rawOptions: CreateOptions = {}): Service { } } path = normalizeSlashes(path); - return outputCache.get(path)?.content || ''; + return ( + outputCache.get(path)?.content || + cachedFsReader.sys.readFile(path) || + '' + ); }, redirectConflictingLibrary: true, onConflictingLibraryRedirect( @@ -784,7 +786,7 @@ export function create(rawOptions: CreateOptions = {}): Service { ) => TypeInfo; const getCanonicalFileName = ((ts as unknown) as TSInternal).createGetCanonicalFileName( - ts.sys.useCaseSensitiveFileNames + cachedFsReader.sys.useCaseSensitiveFileNames ); const moduleTypeClassifier = createModuleTypeClassifier({ @@ -796,7 +798,6 @@ export function create(rawOptions: CreateOptions = {}): Service { if (!transpileOnly) { const fileContents = new Map(); const rootFileNames = new Set(config.fileNames); - const cachedReadFile = cachedLookup(debugFn('readFile', readFile)); // Use language services by default (TODO: invert next major version). if (!options.compilerHost) { @@ -829,7 +830,7 @@ export function create(rawOptions: CreateOptions = {}): Service { // Read contents into TypeScript memory cache. if (contents === undefined) { - contents = cachedReadFile(fileName); + contents = cachedFsReader.sys.readFile(fileName); if (contents === undefined) return; fileVersions.set(fileName, 1); @@ -839,20 +840,15 @@ export function create(rawOptions: CreateOptions = {}): Service { return ts.ScriptSnapshot.fromString(contents); }, - readFile: cachedReadFile, - readDirectory: ts.sys.readDirectory, - getDirectories: cachedLookup( - debugFn('getDirectories', ts.sys.getDirectories) - ), - fileExists: cachedLookup(debugFn('fileExists', fileExists)), - directoryExists: cachedLookup( - debugFn('directoryExists', ts.sys.directoryExists) - ), - realpath: ts.sys.realpath - ? cachedLookup(debugFn('realpath', ts.sys.realpath)) - : undefined, + readFile: cachedFsReader.sys.readFile, + readDirectory: cachedFsReader.sys.readDirectory, + getDirectories: cachedFsReader.sys.getDirectories, + fileExists: cachedFsReader.sys.fileExists, + directoryExists: cachedFsReader.sys.directoryExists, + realpath: cachedFsReader.sys.realpath, getNewLine: () => ts.sys.newLine, - useCaseSensitiveFileNames: () => ts.sys.useCaseSensitiveFileNames, + useCaseSensitiveFileNames: () => + cachedFsReader.sys.useCaseSensitiveFileNames, getCurrentDirectory: () => cwd, getCompilationSettings: () => config.options, getDefaultLibFileName: () => ts.getDefaultLibFilePath(config.options), @@ -877,7 +873,7 @@ export function create(rawOptions: CreateOptions = {}): Service { serviceHost.resolveTypeReferenceDirectives = resolveTypeReferenceDirectives; const registry = ts.createDocumentRegistry( - ts.sys.useCaseSensitiveFileNames, + cachedFsReader.sys.useCaseSensitiveFileNames, cwd ); const service = ts.createLanguageService(serviceHost, registry); @@ -968,27 +964,22 @@ export function create(rawOptions: CreateOptions = {}): Service { }; } else { const sys: _ts.System & _ts.FormatDiagnosticsHost = { + // TODO splat all cachedFsReader methods into here ...ts.sys, ...diagnosticHost, readFile: (fileName: string) => { const cacheContents = fileContents.get(fileName); if (cacheContents !== undefined) return cacheContents; - const contents = cachedReadFile(fileName); + const contents = cachedFsReader.sys.readFile(fileName); if (contents) fileContents.set(fileName, contents); return contents; }, - readDirectory: ts.sys.readDirectory, - getDirectories: cachedLookup( - debugFn('getDirectories', ts.sys.getDirectories) - ), - fileExists: cachedLookup(debugFn('fileExists', fileExists)), - directoryExists: cachedLookup( - debugFn('directoryExists', ts.sys.directoryExists) - ), - resolvePath: cachedLookup(debugFn('resolvePath', ts.sys.resolvePath)), - realpath: ts.sys.realpath - ? cachedLookup(debugFn('realpath', ts.sys.realpath)) - : undefined, + readDirectory: cachedFsReader.sys.readDirectory, + getDirectories: cachedFsReader.sys.getDirectories, + fileExists: cachedFsReader.sys.fileExists, + directoryExists: cachedFsReader.sys.directoryExists, + resolvePath: cachedFsReader.sys.resolvePath, + realpath: cachedFsReader.sys.realpath, }; const host: _ts.CompilerHost = ts.createIncrementalCompilerHost @@ -1297,6 +1288,10 @@ export function create(rawOptions: CreateOptions = {}): Service { addDiagnosticFilter, installSourceMapSupport, enableExperimentalEsmLoaderInterop, + cachedFsReader, + nodeInternalModuleReadJson, + nodePackageJsonReader, + nodeCjsLoaderUtils, }; } @@ -1360,7 +1355,14 @@ function registerExtension( require.extensions[ext] = function (m: any, filename) { if (service.ignored(filename)) return old(m, filename); - assertScriptCanLoadAsCJS(service, m, filename); + // Assert that script can be loaded as CommonJS when we attempt to require it. + // If it should be loaded as ESM, throw ERR_REQUIRE_ESM like node does. + if (engineSupportsPackageTypeField) + service.nodeCjsLoaderUtils!.assertScriptCanLoadAsCJSImpl( + service, + module, + filename + ); const _compile = m._compile; diff --git a/src/module-type-classifier.ts b/src/module-type-classifier.ts index dfe153289..07f8d54ae 100644 --- a/src/module-type-classifier.ts +++ b/src/module-type-classifier.ts @@ -1,4 +1,3 @@ -import { dirname } from 'path'; import { getPatternFromSpec } from './ts-internals'; import { cachedLookup, normalizeSlashes } from './util'; diff --git a/src/repl.ts b/src/repl.ts index 0ef51017d..afc85f5f2 100644 --- a/src/repl.ts +++ b/src/repl.ts @@ -8,13 +8,14 @@ import { start as nodeReplStart, } from 'repl'; import { Context, createContext, Script } from 'vm'; -import { Service, CreateOptions, TSError, env } from './index'; +import { Service, CreateOptions, TSError } from './index'; import { readFileSync, statSync } from 'fs'; import { Console } from 'console'; import * as assert from 'assert'; import type * as tty from 'tty'; import type * as Module from 'module'; import { builtinModules } from 'module'; +import { env } from './util'; // Lazy-loaded. let _processTopLevelAwait: (src: string) => string | null; diff --git a/src/resolver-functions.ts b/src/resolver-functions.ts index e76528155..e83f2e50e 100644 --- a/src/resolver-functions.ts +++ b/src/resolver-functions.ts @@ -65,6 +65,7 @@ export function createResolverFunctions(kwargs: { // .js is switched on-demand if ( resolvedModule.isExternalLibraryImport && + // TODO should include tsx, mts, and cts? ((resolvedFileName.endsWith('.ts') && !resolvedFileName.endsWith('.d.ts')) || isFileKnownToBeInternal(resolvedFileName) || diff --git a/src/util.ts b/src/util.ts index 23e19bba2..f5ba324e1 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,6 +4,10 @@ import { } from 'module'; import type _createRequire from 'create-require'; import * as ynModule from 'yn'; +import type { ProcessEnv } from '.'; + +/** @internal */ +export const env = process.env as ProcessEnv; /** @internal */ export const createRequire = diff --git a/tsconfig.build-schema.json b/tsconfig.build-schema.json index adf48b4c9..bf10488e9 100644 --- a/tsconfig.build-schema.json +++ b/tsconfig.build-schema.json @@ -1,6 +1,7 @@ { "extends": "./tsconfig.json", "compilerOptions": { - "incremental": false + "incremental": false, + "tsBuildInfoFile": null } } diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 000000000..b48f2254b --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,21 @@ +{ + // In our primary tsconfig.json, + // We set rootDir to `.` to encompass both `src` and `dist-raw`, allowing + // us to typecheck against JSDoc annotations in `dist-raw` + + // When building, however, we must set `rootDir` to `src` and disable `allowJs` + // so that `src`->`dist` paths map correctly. + + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "allowJs": false, + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": ".tmp/tsconfig.build.tsbuildinfo", + "rootDirs": [ + ".", + ".tmp/declarations" + ], + } +} diff --git a/tsconfig.declarations.json b/tsconfig.declarations.json new file mode 100644 index 000000000..c1f566ab4 --- /dev/null +++ b/tsconfig.declarations.json @@ -0,0 +1,18 @@ +{ + // Purpose: to extract .d.ts declarations from dist-raw + "extends": "./tsconfig.json", + "include": [ + "src", + "dist-raw" + ], + "compilerOptions": { + "incremental": true, + "allowJs": true, + "emitDeclarationOnly": true, + "stripInternal": false, + "noEmit": false, + "rootDir": ".", + "outDir": ".tmp/declarations", + "tsBuildInfoFile": ".tmp/tsconfig.declarations.tsbuildinfo", + } +} diff --git a/tsconfig.json b/tsconfig.json index bf59b8f83..b492cbd71 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,10 +1,17 @@ { + // Role: support editor code intelligence + + // Note: this tsconfig is not used for compiling. + // See tsconfig.declarations.json and tsconfig.build.json + "$schema": "./tsconfig.schemastore-schema.json", "compilerOptions": { + "allowJs": true, + "noEmit": true, "target": "es2015", "lib": ["es2015", "dom"], - "rootDir": "src", - "outDir": "dist", + "rootDir": ".", + "tsBuildInfoFile": ".tmp/tsconfig.tsbuildinfo", "module": "commonjs", "moduleResolution": "node", "strict": true,