Extract publish registry from publisher

Author: jablko

.github/workflows/publish-registry.yml

@@ -22,7 +22,7 @@ jobs:
           key: cache-${{ github.run_id }}
           restore-keys: cache-
       - name: Publish registry
-        run: yarn workspace @definitelytyped/publisher publish-registry
+        run: node --require source-map-support/register packages/publish-registry/
         env:
           NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
       - if: always()

README.md

@@ -8,6 +8,7 @@ A monorepo for formerly disparate DefinitelyTyped-related tools:
 - [dts-critic](packages/dts-critic): [DefinitelyTyped/dts-critic](https://github.com/DefinitelyTyped/dts-critic)
 - [header-parser](packages/header-parser): [microsoft/definitelytyped-header-parser](https://github.com/microsoft/definitelytyped-header-parser)
 - [perf](packages/perf): [andrewbranch/definitely-not-slow](https://github.com/andrewbranch/definitely-not-slow)
+- [publish-registry](packages/publish-registry): Publish the [`types-registry`](https://www.npmjs.com/package/types-registry) package to npm (a list of all @types packages in the DT repo.
 - [publisher](packages/publisher): the rest of [microsoft/types-publisher](https://github.com/microsoft/types-publisher)
 - [retag](packages/retag): [DefinitelyTyped/dt-retag](https://github.com/DefinitelyTyped/dt-retag)
 - [typescript-versions](packages/typescript-versions): the part of [definitelytyped-header-parser](https://github.com/microsoft/definitelytyped-header-parser) that tracked which TypeScript versions are published to npm and supported on DefinitelyTyped

package.json

@@ -28,6 +28,6 @@
     "ts-jest": "^25.2.1",
     "tslint": "^6.1.2",
     "tslint-microsoft-contrib": "^6.2.0",
-    "typescript": "^4.7.4"
+    "typescript": "^4.6.4"
   }
 }

packages/publish-registry/README.md

@@ -0,0 +1,45 @@
+# publish-registry
+
+[![Publish registry](https://github.com/microsoft/DefinitelyTyped-tools/actions/workflows/publish-registry.yml/badge.svg)](https://github.com/microsoft/DefinitelyTyped-tools/actions/workflows/publish-registry.yml)
+
+Publish the [`types-registry`](https://www.npmjs.com/package/types-registry) package to npm (a list of all @types packages in the DT repo.
+
+## Use
+
+```sh
+yarn workspace @definitelytyped/publisher parse
+node packages/publish-registry/
+```
+
+1. [Parse declarations](../publisher/README.md#parse-the-definitions).
+2. Run this package's script.
+
+### Options
+
+<dl><dt>
+
+`--dry`
+
+</dt><dd>
+
+Don't actually publish the package, just show what would be done.
+
+</dd></dl>
+
+### Environment variables
+
+<dl><dt>
+
+[`NPM_TOKEN`](https://docs.npmjs.com/about-access-tokens)
+
+</dt><dd>
+
+Not required for a dry run.
+Only used to actually publish the package.
+
+</dd></dl>
+
+## Logs
+
+GitHub Actions runs this package's script weekly.
+You can [examine the logs](https://github.com/microsoft/DefinitelyTyped-tools/actions/workflows/publish-registry.yml).

packages/publish-registry/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@definitelytyped/publish-registry",
+  "version": "1.0.0",
+  "description": "Publish the types-registry package to npm (a list of all @types packages in the DT repo).",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc --build"
+  },
+  "dependencies": {
+    "@definitelytyped/definitions-parser": "^0.0.121",
+    "@definitelytyped/utils": "^0.0.121",
+    "fs-extra": "^10.1.0",
+    "pacote": "^13.6.1",
+    "semver": "^7.3.7",
+    "yargs": "^17.5.1"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^9.0.13",
+    "@types/pacote": "^11.1.5",
+    "@types/semver": "^7.3.10",
+    "@types/yargs": "^17.0.10"
+  },
+  "private": true
+}

packages/publisher/src/publish-registry.ts renamed to packages/publish-registry/src/index.ts

@@ -3,8 +3,6 @@ import process from "process";
 import { emptyDir } from "fs-extra";
 import * as yargs from "yargs";
 
-import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
-import { outputDirPath, validateOutputPath } from "./lib/settings";
 import {
   getDefinitelyTyped,
   AllPackages,
@@ -33,21 +31,18 @@ import {
 } from "@definitelytyped/utils";
 import * as pacote from "pacote";
 import * as semver from "semver";
-// @ts-ignore
-import pkg from "../package.json";
 
-const typesRegistry = "types-registry";
-const registryOutputPath = joinPaths(outputDirPath, typesRegistry);
+const registryOutputPath = joinPaths(__dirname, "..", "output", "types-registry");
 const readme = `This package contains a listing of all packages published to the @types scope on NPM.
-Generated by [types-publisher](${pkg.homepage}).`;
+Generated by [types-publisher](https://github.com/microsoft/DefinitelyTyped-tools/tree/master/packages/publish-registry#readme).`;
 
 if (!module.parent) {
-  const dry = !!yargs.argv.dry;
+  const { dry } = yargs.argv as never;
   logUncaughtErrors(async () => {
-    const dt = await getDefinitelyTyped(
-      process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions,
-      loggerWithErrors()[0]
-    );
+    const options = process.env.GITHUB_ACTIONS
+      ? { definitelyTypedPath: undefined, progress: false, parseInParallel: false }
+      : { definitelyTypedPath: "../../../DefinitelyTyped", progress: true, parseInParallel: true };
+    const dt = await getDefinitelyTyped(options, loggerWithErrors()[0]);
     await publishRegistry(dt, await AllPackages.read(dt), dry);
   });
 }
@@ -56,7 +51,7 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages,
   const [log, logResult] = logger();
   log("=== Publishing types-registry ===");
 
-  const { latestVersion, maxVersion, latestContentHash } = await fetchAndProcessNpmInfo(typesRegistry);
+  const { latestVersion, maxVersion, latestContentHash } = await fetchAndProcessNpmInfo("types-registry");
   assert(semver.satisfies(latestVersion, "~0.1"));
 
   // Don't include not-needed packages in the registry.
@@ -69,7 +64,7 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages,
   await writeLog("publish-registry.md", logResult());
 
   async function publishToRegistry() {
-    const packageJson = generatePackageJson(typesRegistry, newVersion, newContentHash);
+    const packageJson = generatePackageJson("types-registry", newVersion, newContentHash);
     await generate(registry, packageJson);
 
     const token = process.env.NPM_TOKEN!;
@@ -81,10 +76,10 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages,
       // If so, we should just update it to "latest" now.
       log("Old version of types-registry was never tagged latest, so updating");
       await validateIsSubset(readNotNeededPackages(dt), log);
-      await (await publishClient()).tag(typesRegistry, maxVersion, "latest", dry, log);
+      await (await publishClient()).tag("types-registry", maxVersion, "latest", dry, log);
     } else if (latestContentHash !== newContentHash) {
       log("New packages have been added, so publishing a new registry.");
-      await publish(await publishClient(), typesRegistry, packageJson, newVersion, dry, log);
+      await publish(await publishClient(), "types-registry", packageJson, newVersion, dry, log);
     } else {
       log("No new packages published, so no need to publish new registry.");
       // Just making sure...
@@ -133,6 +128,8 @@ async function publish(
   await client.tag(packageName, version, "latest", dry, log);
 }
 
+const validateOutputPath = joinPaths(__dirname, "..", "validateOutput");
+
 async function installForValidate(log: Logger): Promise<void> {
   await emptyDir(validateOutputPath);
   await writeJson(joinPaths(validateOutputPath, "package.json"), {
@@ -164,9 +161,8 @@ async function validate(log: Logger): Promise<void> {
 
 async function validateIsSubset(notNeeded: readonly NotNeededPackage[], log: Logger): Promise<void> {
   await installForValidate(log);
-  const indexJson = "index.json";
-  const actual = (await readJson(joinPaths(validateTypesRegistryPath, indexJson))) as Registry;
-  const expected = (await readJson(joinPaths(registryOutputPath, indexJson))) as Registry;
+  const actual = (await readJson(joinPaths(validateTypesRegistryPath, "index.json"))) as Registry;
+  const expected = (await readJson(joinPaths(registryOutputPath, "index.json"))) as Registry;
   for (const key of Object.keys(actual.entries)) {
     if (!(key in expected.entries) && !notNeeded.some((p) => p.name === key)) {
       throw new Error(`Actual types-registry has unexpected key ${key}`);
@@ -206,17 +202,21 @@ function assertJsonNewer(newer: { [s: string]: any }, older: { [s: string]: any
 }
 
 function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): object {
-  const json = {
+  return {
     name,
     version,
     description: "A registry of TypeScript declaration file packages published within the @types scope.",
-    repository: pkg.repository,
     keywords: ["TypeScript", "declaration", "files", "types", "packages"],
-    author: "Microsoft Corp.",
+    homepage: "https://github.com/microsoft/DefinitelyTyped-tools/tree/master/packages/publish-registry#readme",
     license: "MIT",
+    author: "Microsoft Corp.",
+    repository: {
+      type: "git",
+      url: "https://github.com/microsoft/DefinitelyTyped-tools.git",
+      directory: "packages/publish-registry",
+    },
     typesPublisherContentHash,
   };
-  return json;
 }
 
 interface Registry {
@@ -256,5 +256,6 @@ async function fetchAndProcessNpmInfo(packageName: string): Promise<ProcessedNpm
   const latestVersion = info["dist-tags"].latest;
   const maxVersion = semver.maxSatisfying(Object.keys(info.versions), "*");
   assert.strictEqual(maxVersion, info["dist-tags"].next);
-  return { latestVersion, maxVersion, latestContentHash: info.versions[latestVersion].typesPublisherContentHash };
+  const latestContentHash = info.versions[latestVersion].typesPublisherContentHash;
+  return { latestVersion, maxVersion, latestContentHash };
 }

packages/publish-registry/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src/",
+    "outDir": "dist/"
+  },
+  "references": [{ "path": "../definitions-parser/" }, { "path": "../utils/" }]
+}

packages/publisher/README.md

@@ -43,7 +43,6 @@ npm run upload-blobs
 and optionally (in production, these run once a week):
 
 ```
-npm run publish-registry
 npm run validate
 ```
 
@@ -241,13 +240,6 @@ It also indicates any errors that may have occurred during publishing.
 Note that unlike other steps, this log file output is *not* idempotent.
 Scripts should save this log under a unique filename so any errors may be reviewed.
 
-# Publish registry
-
-> `npm run publish -- [--dry]`
-
-This step publishes the `types-registry` package on NPM, which keeps a list of all `@types` packages.
-This step only happens if there are some new packages to register.
-
 # Settings
 
 This file contains settings used by the publisher.

packages/publisher/package.json

@@ -48,7 +48,6 @@
     "validate": "node -r source-map-support/register dist/validate.js",
     "publish-packages": "node -r source-map-support/register dist/publish-packages.js",
     "publish-packages-dry": "node -r source-map-support/register dist/publish-packages.js --dry",
-    "publish-registry": "node -r source-map-support/register dist/publish-registry.js",
     "full": "node -r source-map-support/register dist/full.js",
     "full-dry": "node -r source-map-support/register dist/full.js --dry",
     "test-tsNext": "node -r source-map-support/register dist/tester/test.js --all --tsNext"

packages/publisher/src/clean.ts

@@ -1,7 +1,7 @@
 import { clean as cleanParser } from "@definitelytyped/definitions-parser";
 import { cleanLogDirectory } from "@definitelytyped/utils";
 import { removeSync } from "fs-extra";
-import { outputDirPath, validateOutputPath } from "./lib/settings";
+import { outputDirPath } from "./lib/settings";
 
 if (!module.parent) {
   clean();
@@ -11,5 +11,4 @@ export function clean() {
   cleanParser();
   cleanLogDirectory();
   removeSync(outputDirPath);
-  removeSync(validateOutputPath);
 }