Publish older versions without touching latest tag

Author: jablko

package.json

@@ -28,6 +28,6 @@
     "ts-jest": "^25.2.1",
     "tslint": "^6.1.2",
     "tslint-microsoft-contrib": "^6.2.0",
-    "typescript": "^4.5.5"
+    "typescript": "^4.6.4"
   }
 }

packages/publisher/package.json

@@ -19,18 +19,23 @@
     "fs-extra": "^9.1.0",
     "fstream": "^1.0.12",
     "hh-mm-ss": "^1.2.0",
+    "libnpmpublish": "^6.0.4",
     "longjohn": "^0.2.11",
     "oboe": "^2.1.3",
+    "pacote": "^13.6.0",
     "semver": "^7.3.7",
     "source-map-support": "^0.4.0",
     "typescript": "^4.1.0",
     "yargs": "15.3.1"
   },
   "devDependencies": {
+    "@npm/types": "^1.0.2",
     "@types/fs-extra": "^9.0.8",
     "@types/hh-mm-ss": "^1.2.1",
+    "@types/libnpmpublish": "^4.0.3",
     "@types/mz": "^0.0.31",
     "@types/oboe": "^2.0.28",
+    "@types/pacote": "^11.1.4",
     "@types/source-map-support": "^0.4.0",
     "@types/yargs": "^15.0.4"
   },

packages/publisher/src/calculate-versions.ts

@@ -46,9 +46,7 @@ async function computeAndSaveChangedPackages(
 ): Promise<ChangedPackages> {
   const cp = await computeChangedPackages(allPackages, log, client);
   const json: ChangedPackagesJson = {
-    changedTypings: cp.changedTypings.map(
-      ({ pkg: { id }, version, latestVersion }): ChangedTypingJson => ({ id, version, latestVersion })
-    ),
+    changedTypings: cp.changedTypings.map(({ pkg: { id }, version }): ChangedTypingJson => ({ id, version })),
     changedNotNeededPackages: cp.changedNotNeededPackages.map((p) => p.name),
   };
   await writeDataFile(versionsFilename, json);
@@ -71,10 +69,7 @@ async function computeChangedPackages(
           `'${pkg.name}' depends on '${name}' which does not exist on npm. All dependencies must exist.`
         );
       }
-      const latestVersion = pkg.isLatest
-        ? undefined
-        : (await fetchTypesPackageVersionInfo(allPackages.getLatest(pkg), client, /*publish*/ true)).version;
-      return { pkg, version, latestVersion };
+      return { pkg, version };
     }
     return undefined;
   });

packages/publisher/src/generate-packages.ts

@@ -17,7 +17,6 @@ import {
   writeLog,
   writeFile,
   Logger,
-  writeTgz,
   withNpmCache,
   UncachedNpmInfoClient,
   CachedNpmInfoClient,
@@ -34,6 +33,7 @@ import {
   License,
   formatTypingVersion,
 } from "@definitelytyped/definitions-parser";
+import * as pacote from "pacote";
 import { readChangedPackages, ChangedPackages } from "./lib/versions";
 import { outputDirectory } from "./util/util";
 import { skipBadPublishes } from "./lib/npm";
@@ -65,7 +65,7 @@ export default async function generatePackages(
   for (const { pkg, version } of changedPackages.changedTypings) {
     await generateTypingPackage(pkg, allPackages, version, dt);
     if (tgz) {
-      await writeTgz(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
+      await pacote.tarball.file(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
     }
     log(` * ${pkg.desc}`);
   }

packages/publisher/src/lib/package-publisher.ts

@@ -1,42 +1,51 @@
-import assert = require("assert");
-import { Logger, joinPaths, readFileAndWarn, NpmPublishClient } from "@definitelytyped/utils";
+import { Logger, NpmPublishClient } from "@definitelytyped/utils";
 import { NotNeededPackage, AnyPackage } from "@definitelytyped/definitions-parser";
-import { updateTypeScriptVersionTags, updateLatestTag } from "@definitelytyped/retag";
+import { updateTypeScriptVersionTags } from "@definitelytyped/retag";
+import * as libpub from "libnpmpublish";
+import * as pacote from "pacote";
 import { ChangedTyping } from "./versions";
 import { outputDirectory } from "../util/util";
 
+// https://github.com/npm/types/pull/18
+declare module "libnpmpublish" {
+  function publish(
+    manifest: Omit<import("@npm/types").PackageJson, "bundledDependencies">,
+    tarData: Buffer,
+    opts?: import("npm-registry-fetch").Options
+  ): Promise<Response>;
+}
+
 export async function publishTypingsPackage(
   client: NpmPublishClient,
   changedTyping: ChangedTyping,
+  token: string,
   dry: boolean,
   log: Logger
 ): Promise<void> {
-  const { pkg, version, latestVersion } = changedTyping;
-  await common(client, pkg, log, dry);
+  const { pkg, version } = changedTyping;
+  await common(pkg, token, dry);
   if (pkg.isLatest) {
     await updateTypeScriptVersionTags(pkg, version, client, log, dry);
   }
-  assert((latestVersion === undefined) === pkg.isLatest);
-  if (latestVersion !== undefined) {
-    // If this is an older version of the package, we still update tags for the *latest*.
-    // NPM will update "latest" even if we are publishing an older version of a package (https://github.com/npm/npm/issues/6778),
-    // so we must undo that by re-tagging latest.
-    await updateLatestTag(pkg.fullNpmName, latestVersion, client, log, dry);
-  }
 }
 
 export async function publishNotNeededPackage(
-  client: NpmPublishClient,
   pkg: NotNeededPackage,
+  token: string,
   dry: boolean,
   log: Logger
 ): Promise<void> {
   log(`Deprecating ${pkg.name}`);
-  await common(client, pkg, log, dry);
+  await common(pkg, token, dry);
 }
 
-async function common(client: NpmPublishClient, pkg: AnyPackage, log: Logger, dry: boolean): Promise<void> {
+async function common(pkg: AnyPackage, token: string, dry: boolean): Promise<void> {
   const packageDir = outputDirectory(pkg);
-  const packageJson = await readFileAndWarn("generate", joinPaths(packageDir, "package.json"));
-  await client.publish(packageDir, packageJson, dry, log);
+  const manifest = await pacote.manifest(packageDir).catch((reason) => {
+    throw reason.code === "ENOENT" ? new Error("Run generate first!", { cause: reason }) : reason;
+  });
+  const tarData = await pacote.tarball(packageDir);
+  // Make sure we never assign the latest tag to an older version.
+  if (!dry)
+    await libpub.publish(manifest, tarData, { defaultTag: pkg.isLatest ? "latest" : "", access: "public", token });
 }

packages/publisher/src/lib/versions.ts

@@ -13,8 +13,6 @@ export interface ChangedTyping {
   readonly pkg: TypingsData;
   /** This is the version to be published, meaning it's the version that doesn't exist yet. */
   readonly version: string;
-  /** For a non-latest version, this is the latest version; publishing an old version updates the 'latest' tag and we want to change it back. */
-  readonly latestVersion: string | undefined;
 }
 
 export interface ChangedPackagesJson {
@@ -25,7 +23,6 @@ export interface ChangedPackagesJson {
 export interface ChangedTypingJson {
   readonly id: PackageId;
   readonly version: string;
-  readonly latestVersion?: string;
 }
 
 export interface ChangedPackages {
@@ -37,10 +34,9 @@ export async function readChangedPackages(allPackages: AllPackages): Promise<Cha
   const json = (await readDataFile("calculate-versions", versionsFilename)) as ChangedPackagesJson;
   return {
     changedTypings: json.changedTypings.map(
-      ({ id, version, latestVersion }): ChangedTyping => ({
+      ({ id, version }): ChangedTyping => ({
         pkg: allPackages.getTypingsData(id),
         version,
-        latestVersion,
       })
     ),
     changedNotNeededPackages: json.changedNotNeededPackages.map((id) =>

packages/publisher/src/publish-packages.ts

@@ -45,12 +45,13 @@ export default async function publishPackages(
     log("=== Publishing packages ===");
   }
 
-  const client = await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined);
+  const token = await getSecret(Secret.NPM_TOKEN);
+  const client = await NpmPublishClient.create(token, undefined);
 
   for (const cp of changedPackages.changedTypings) {
     log(`Publishing ${cp.pkg.desc}...`);
 
-    await publishTypingsPackage(client, cp, dry, log);
+    await publishTypingsPackage(client, cp, token, dry, log);
 
     const commits = (await queryGithub(
       `repos/DefinitelyTyped/DefinitelyTyped/commits?path=types%2f${cp.pkg.subDirectoryPath}`,
@@ -134,7 +135,7 @@ export default async function publishPackages(
     async (infoClient) => {
       for (const n of changedPackages.changedNotNeededPackages) {
         const target = skipBadPublishes(n, infoClient, log);
-        await publishNotNeededPackage(client, target, dry, log);
+        await publishNotNeededPackage(target, token, dry, log);
       }
     },
     cacheDirPath

packages/publisher/src/publish-registry.ts

@@ -34,6 +34,9 @@ import {
   isObject,
   max,
 } from "@definitelytyped/utils";
+import { PackageJson } from "@npm/types";
+import * as libpub from "libnpmpublish";
+import * as pacote from "pacote";
 import * as semver from "semver";
 import { getSecret, Secret } from "./lib/secrets";
 // @ts-ignore
@@ -88,7 +91,7 @@ export default async function publishRegistry(
 
     const token = await getSecret(Secret.NPM_TOKEN);
 
-    const publishClient = () => NpmPublishClient.create(token, { defaultTag: "next" });
+    const publishClient = () => NpmPublishClient.create(token);
     if (!semver.eq(highestSemverVersion, npmVersion)) {
       // There was an error in the last publish and types-registry wasn't validated.
       // This may have just been due to a timeout, so test if types-registry@next is a subset of the one we're about to publish.
@@ -98,7 +101,7 @@ export default async function publishRegistry(
       await (await publishClient()).tag(typesRegistry, String(highestSemverVersion), "latest", dry, log);
     } else if (npmContentHash !== newContentHash && isTimeForNewVersion) {
       log("New packages have been added, so publishing a new registry.");
-      await publish(await publishClient(), typesRegistry, packageJson, newVersion, dry, log);
+      await publish(await publishClient(), packageJson, token, dry, log);
     } else {
       const reason =
         npmContentHash === newContentHash ? "No new packages published" : "Was modified less than a week ago";
@@ -137,13 +140,13 @@ async function generate(registry: string, packageJson: {}): Promise<void> {
 
 async function publish(
   client: NpmPublishClient,
-  packageName: string,
-  packageJson: {},
-  version: string,
+  packageJson: PackageJson,
+  token: string,
   dry: boolean,
   log: Logger
 ): Promise<void> {
-  await client.publish(registryOutputPath, packageJson, dry, log);
+  const tarData = await pacote.tarball(registryOutputPath);
+  if (!dry) await libpub.publish(packageJson, tarData, { defaultTag: "next", access: "public", token });
   // Sleep for 60 seconds to let NPM update.
   if (dry) {
     log("(dry) Skipping 60 second sleep...");
@@ -153,7 +156,7 @@ async function publish(
   }
   // Don't set it as "latest" until *after* it's been validated.
   await validate(log);
-  await client.tag(packageName, version, "latest", dry, log);
+  await client.tag(packageJson.name, packageJson.version, "latest", dry, log);
 }
 
 async function installForValidate(log: Logger): Promise<void> {
@@ -228,7 +231,7 @@ function assertJsonNewer(newer: { [s: string]: any }, older: { [s: string]: any
   }
 }
 
-function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): object {
+function generatePackageJson(name: string, version: string, typesPublisherContentHash: string): PackageJson {
   const json = {
     name,
     version,

packages/publisher/test/package-publisher.test.ts

@@ -0,0 +1,47 @@
+import { NotNeededPackage, TypingsData } from "@definitelytyped/definitions-parser";
+import * as libpub from "libnpmpublish";
+import { publishNotNeededPackage, publishTypingsPackage } from "../src/lib/package-publisher";
+
+jest.mock("libnpmpublish");
+jest.mock("pacote", () => ({ async manifest() {}, async tarball() {} }));
+
+const client = { async tag() {} };
+
+const latestVersion = {
+  pkg: new TypingsData({ typingsPackageName: "some-package" } as never, /*isLatest*/ true),
+  version: "1.2.3",
+};
+const olderVersion = {
+  pkg: new TypingsData({ typingsPackageName: "some-package" } as never, /*isLatest*/ false),
+  version: "1.2.3",
+};
+const notNeeded = new NotNeededPackage("not-needed", "not-needed", "1.2.3");
+
+function log() {}
+
+test("Latest version gets latest tag", async () => {
+  await publishTypingsPackage(client as never, latestVersion, /*token*/ "", /*dry*/ false, log);
+  expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
+    defaultTag: "latest",
+    access: "public",
+    token: "",
+  });
+});
+
+test("Publishing older versions doesn't change the latest tag", async () => {
+  await publishTypingsPackage(client as never, olderVersion, /*token*/ "", /*dry*/ false, log);
+  expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
+    defaultTag: "",
+    access: "public",
+    token: "",
+  });
+});
+
+test("Not-needed types package gets latest tag", async () => {
+  await publishNotNeededPackage(notNeeded, /*token*/ "", /*dry*/ false, log);
+  expect(libpub.publish).toHaveBeenLastCalledWith(/*manifest*/ undefined, /*tarData*/ undefined, {
+    defaultTag: "latest",
+    access: "public",
+    token: "",
+  });
+});

packages/utils/.gitignore

@@ -1,2 +1 @@
 logs
-test/data/pack.tgz