Merge pull request #981 from Aukevanoost/f/performance-caching-II

feat(nf): Performance boost - cache external artifacts with checksum

Author: Aukevanoost

libs/native-federation-core/src/lib/core/build-for-federation.ts

@@ -14,6 +14,8 @@ import { FederationOptions } from './federation-options';
 import { writeFederationInfo } from './write-federation-info';
 import { writeImportMap } from './write-import-map';
 import { logger } from '../utils/logger';
+import { getCachePath } from './bundle-caching';
+import { normalizePackageName } from '../utils/normalize';
 import { AbortedError } from '../utils/errors';
 
 export interface BuildParams {
@@ -27,9 +29,7 @@ export const defaultBuildParams: BuildParams = {
   skipShared: false,
 };
 
-// Externals cache
 const sharedPackageInfoCache: SharedInfo[] = [];
-const cachedSharedPackages = new Set<string>();
 
 export async function buildForFederation(
   config: NormalizedFederationConfig,
@@ -64,7 +64,23 @@ export async function buildForFederation(
     ? describeExposed(config, fedOptions)
     : artefactInfo.exposes;
 
-  if (!buildParams.skipShared) {
+  const cacheProjectFolder = normalizePackageName(config.name);
+  if (cacheProjectFolder.length < 1) {
+    logger.warn(
+      "Project name in 'federation.config.js' is empty, defaulting to root cache folder (could collide with other projects in the workspace).",
+    );
+  }
+
+  const pathToCache = getCachePath(
+    fedOptions.workspaceRoot,
+    cacheProjectFolder,
+  );
+
+  if (!buildParams.skipShared && sharedPackageInfoCache.length > 0) {
+    logger.info('Checksum matched, re-using cached externals.');
+  }
+
+  if (!buildParams.skipShared && sharedPackageInfoCache.length === 0) {
     const { sharedBrowser, sharedServer, separateBrowser, separateServer } =
       splitShared(config.shared);
 
@@ -76,6 +92,7 @@ export async function buildForFederation(
         fedOptions,
         externals,
         'browser',
+        { pathToCache, bundleName: 'browser-shared' },
       );
 
       logger.measure(
@@ -84,9 +101,7 @@ export async function buildForFederation(
       );
 
       sharedPackageInfoCache.push(...sharedPackageInfoBrowser);
-      Object.keys(sharedBrowser).forEach((packageName) =>
-        cachedSharedPackages.add(packageName),
-      );
+
       if (signal?.aborted)
         throw new AbortedError(
           '[buildForFederation] After shared-browser bundle',
@@ -101,15 +116,14 @@ export async function buildForFederation(
         fedOptions,
         externals,
         'node',
+        { pathToCache, bundleName: 'node-shared' },
       );
       logger.measure(
         start,
         '[build artifacts] - To bundle all shared node externals',
       );
       sharedPackageInfoCache.push(...sharedPackageInfoServer);
-      Object.keys(sharedServer).forEach((packageName) =>
-        cachedSharedPackages.add(packageName),
-      );
+
       if (signal?.aborted)
         throw new AbortedError('[buildForFederation] After shared-node bundle');
     }
@@ -122,15 +136,14 @@ export async function buildForFederation(
         config,
         fedOptions,
         'browser',
+        pathToCache,
       );
       logger.measure(
         start,
         '[build artifacts] - To bundle all separate browser externals',
       );
       sharedPackageInfoCache.push(...separatePackageInfoBrowser);
-      Object.keys(separateBrowser).forEach((packageName) =>
-        cachedSharedPackages.add(packageName),
-      );
+
       if (signal?.aborted)
         throw new AbortedError(
           '[buildForFederation] After separate-browser bundle',
@@ -145,15 +158,13 @@ export async function buildForFederation(
         config,
         fedOptions,
         'node',
+        pathToCache,
       );
       logger.measure(
         start,
         '[build artifacts] - To bundle all separate node externals',
       );
       sharedPackageInfoCache.push(...separatePackageInfoServer);
-      Object.keys(separateServer).forEach((packageName) =>
-        cachedSharedPackages.add(packageName),
-      );
     }
 
     if (signal?.aborted)
@@ -203,6 +214,7 @@ async function bundleSeparate(
   config: NormalizedFederationConfig,
   fedOptions: FederationOptions,
   platform: 'node' | 'browser',
+  pathToCache: string,
 ) {
   const bundlePromises = Object.entries(separateBrowser).map(
     async ([key, shared]) => {
@@ -216,6 +228,10 @@ async function bundleSeparate(
         fedOptions,
         filteredExternals,
         platform,
+        {
+          pathToCache,
+          bundleName: `${platform}-${normalizePackageName(key)}`,
+        },
       );
     },
   );
@@ -233,7 +249,6 @@ function splitShared(
   const separateServer: Record<string, NormalizedSharedConfig> = {};
 
   for (const key in shared) {
-    if (cachedSharedPackages.has(key)) continue;
     const obj = shared[key];
     if (obj.platform === 'node' && obj.build === 'default') {
       sharedServer[key] = obj;

libs/native-federation-core/src/lib/core/bundle-caching.ts

@@ -0,0 +1,115 @@
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import { NormalizedSharedConfig } from '../config/federation-config';
+import { SharedInfo } from '@softarc/native-federation-runtime';
+import { logger } from '../utils/logger';
+
+export const getCachePath = (workspaceRoot: string, project: string) =>
+  path.join(workspaceRoot, 'node_modules/.cache/native-federation', project);
+
+export const getFilename = (title: string) => {
+  return `${title}.meta.json`;
+};
+
+export const getChecksum = (
+  shared: Record<string, NormalizedSharedConfig>,
+): string => {
+  const denseExternals = Object.keys(shared)
+    .sort()
+    .reduce((clean, external) => {
+      return (
+        clean +
+        ':' +
+        external +
+        (shared[external].version ? `@${shared[external].version}` : '')
+      );
+    }, 'deps');
+
+  return crypto.createHash('sha256').update(denseExternals).digest('hex');
+};
+
+export const cacheEntry = (pathToCache: string, fileName: string) => ({
+  getMetadata: (
+    checksum: string,
+  ):
+    | {
+        checksum: string;
+        externals: SharedInfo[];
+        files: string[];
+      }
+    | undefined => {
+    const metadataFile = path.join(pathToCache, fileName);
+    if (!fs.existsSync(pathToCache) || !fs.existsSync(metadataFile))
+      return undefined;
+
+    const cachedResult: {
+      checksum: string;
+      externals: SharedInfo[];
+      files: string[];
+    } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8'));
+    if (cachedResult.checksum !== checksum) return undefined;
+    return cachedResult;
+  },
+  persist: (payload: {
+    checksum: string;
+    externals: SharedInfo[];
+    files: string[];
+  }) => {
+    fs.writeFileSync(
+      path.join(pathToCache, fileName),
+      JSON.stringify(payload),
+      'utf-8',
+    );
+  },
+  copyFiles: (fullOutputPath: string) => {
+    const metadataFile = path.join(pathToCache, fileName);
+    if (!fs.existsSync(metadataFile))
+      throw new Error(
+        'Error copying artifacts to dist, metadata file could not be found.',
+      );
+
+    const cachedResult: {
+      externals: SharedInfo[];
+      files: string[];
+    } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8'));
+
+    fs.mkdirSync(path.dirname(fullOutputPath), { recursive: true });
+
+    cachedResult.files.forEach((file) => {
+      const cachedFile = path.join(pathToCache, file);
+      const distFileName = path.join(fullOutputPath, file);
+
+      if (fs.existsSync(cachedFile)) {
+        fs.copyFileSync(cachedFile, distFileName);
+      }
+    });
+  },
+  clear: () => {
+    const metadataFile = path.join(pathToCache, fileName);
+    if (!fs.existsSync(pathToCache)) {
+      fs.mkdirSync(pathToCache, { recursive: true });
+      logger.debug(`Creating cache folder '${pathToCache}' for '${fileName}'.`);
+      return;
+    }
+    if (!fs.existsSync(metadataFile)) {
+      logger.debug(
+        `Could not purge cached bundle, metadata file '${metadataFile}' does not exist.`,
+      );
+      return;
+    }
+
+    const cachedResult: {
+      checksum: string;
+      externals: SharedInfo[];
+      files: string[];
+    } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8'));
+
+    cachedResult.files.forEach((file) => {
+      const cachedFile = path.join(pathToCache, file);
+      if (fs.existsSync(cachedFile)) fs.unlinkSync(cachedFile);
+    });
+
+    fs.unlinkSync(metadataFile);
+  },
+});