Use content hash vs. freshness in publish registry

Author: jablko

.github/workflows/ci.yml

@@ -36,6 +36,11 @@ jobs:
       run: yarn lint
     - name: test
       run: yarn test
+  publish-registry-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/publish-registry.yml
+    with:
+      dry-run: true
   publish_alpha:
     name: publish alpha release
     runs-on: ubuntu-latest

.github/workflows/publish-registry.yml

@@ -3,7 +3,14 @@ on:
   schedule:
     # https://crontab.guru/#0_0_*_*_0
     - cron: 0 0 * * 0
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
   workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
 jobs:
   publish-registry:
     runs-on: ubuntu-latest
@@ -21,8 +28,8 @@ jobs:
           path: packages/utils/cache/
           key: cache-${{ github.run_id }}
           restore-keys: cache-
-      - name: Publish registry
-        run: yarn workspace @definitelytyped/publisher publish-registry
+      - name: Publish registry${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn workspace @definitelytyped/publisher publish-registry${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
         env:
           NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
       - if: always()

packages/publisher/package.json

@@ -20,6 +20,7 @@
     "fstream": "^1.0.12",
     "hh-mm-ss": "^1.2.0",
     "longjohn": "^0.2.11",
+    "npm-pick-manifest": "^7.0.1",
     "pacote": "^13.6.1",
     "semver": "^7.3.7",
     "source-map-support": "^0.4.0",

packages/publisher/src/publish-registry.ts

@@ -1,6 +1,8 @@
 import assert = require("assert");
 import process from "process";
 import { emptyDir } from "fs-extra";
+// @ts-expect-error
+import pickManifest from "npm-pick-manifest";
 import * as yargs from "yargs";
 
 import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
@@ -230,10 +232,7 @@ async function generateRegistry(typings: readonly TypingsData[]): Promise<Regist
   return {
     entries: Object.fromEntries(
       await Promise.all(
-        typings.map(async (typing) => [
-          typing.name,
-          filterTags((await pacote.packument(typing.fullNpmName, { cache: cacheDir }))["dist-tags"]),
-        ])
+        typings.map(async (typing) => [typing.name, filterTags((await revalidate(typing))["dist-tags"])])
       )
     ),
   };
@@ -245,6 +244,27 @@ async function generateRegistry(typings: readonly TypingsData[]): Promise<Regist
   }
 }
 
+/** Fetch packument from the cache or the network, depending on whether the types have changed or not. */
+async function revalidate(typing: TypingsData) {
+  const offline = await pacote
+    .packument(typing.fullNpmName, { cache: cacheDir, fullMetadata: true, offline: true })
+    .catch((reason) => {
+      if (reason.code !== "ENOTCACHED") throw reason;
+      return undefined;
+    });
+  try {
+    if (
+      offline &&
+      pickManifest(offline, `~${typing.major}.${typing.minor}`).typesPublisherContentHash === typing.contentHash
+    )
+      return offline;
+    // @ts-expect-error
+  } catch (reason: {}) {
+    if (reason.code !== "ETARGET") throw reason;
+  }
+  return pacote.packument(typing.fullNpmName, { cache: cacheDir, fullMetadata: true, preferOnline: true });
+}
+
 interface ProcessedNpmInfo {
   readonly latestVersion: string;
   readonly maxVersion: string;

yarn.lock

@@ -6952,7 +6952,7 @@ npm-pick-manifest@^6.0.0, npm-pick-manifest@^6.1.1:
     npm-package-arg "^8.1.2"
     semver "^7.3.4"
 
-npm-pick-manifest@^7.0.0:
+npm-pick-manifest@^7.0.0, npm-pick-manifest@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/npm-pick-manifest/-/npm-pick-manifest-7.0.1.tgz#76dda30a7cd6b99be822217a935c2f5eacdaca4c"
   integrity sha512-IA8+tuv8KujbsbLQvselW2XQgmXWS47t3CB0ZrzsRZ82DbDfkcFunOaPm4X7qNuhMfq+FmV7hQT4iFVpHqV7mg==