add ability to generate chained sourcemap for precompiled sources

Signed-off-by: Karthik Ganeshram <[email protected]>

Author: karthik2804

packages/build-tools/package-lock.json

@@ -29,13 +29,14 @@
     "typescript": "^5.7.3"
   },
   "dependencies": {
+    "@ampproject/remapping": "^2.3.0",
     "@bytecodealliance/componentize-js": "^0.18.1",
     "@bytecodealliance/jco": "^1.10.2",
-    "yargs": "^17.7.2",
     "acorn-walk": "^8.3.4",
     "acron": "^1.0.5",
     "magic-string": "^0.30.17",
-    "regexpu-core": "^6.2.0"
+    "regexpu-core": "^6.2.0",
+    "yargs": "^17.7.2"
   },
   "files": [
     "lib",
@@ -53,4 +54,4 @@
       }
     ]
   }
-}
+}

packages/build-tools/package.json

@@ -5,6 +5,8 @@ import { version as componentizeVersion } from '@bytecodealliance/componentize-j
 import { getPackagesWithWasiDeps, processWasiDeps } from './wasiDepsParser.js';
 import {
   calculateChecksum,
+  chainSourceMaps,
+  fileExists,
   saveBuildData,
 } from './utils.js';
 import { getCliArgs } from './cli.js';
@@ -14,6 +16,7 @@ import { mergeWit } from '../lib/wit_tools.js';
 //@ts-ignore
 import { precompile } from "./precompile.js"
 import path from 'node:path'
+import { SourceMapInput } from '@ampproject/remapping';
 
 async function main() {
   try {
@@ -65,13 +68,20 @@ async function main() {
     console.log('Componentizing...');
 
     const source = await readFile(src, 'utf8');
-    const precompiledSource = precompile(source, src, true) as string;
+    let { content: precompiledSource, sourceMapInput: precompiledSourceMap } = precompile(source, src, true, 'precompiled-source.js') as { content: string; sourceMapInput: SourceMapInput };
+    // Check if input file has a source map because if we does, we need to chain it with the precompiled source map
+    if (await fileExists(src + '.map')) {
+      const inputSourceMap = JSON.parse(await readFile(src + '.map', 'utf8')) as SourceMapInput;
+      precompiledSourceMap = chainSourceMaps(precompiledSourceMap, { src: inputSourceMap }) as SourceMapInput;
+    }
 
-    // Write precompiled source to disk for debugging purposes In the future we
-    // will also write a source map to make debugging easier
+    // Write precompiled source to disk for debugging purposes.
     let srcDir = path.dirname(src);
     let precompiledSourcePath = path.join(srcDir, 'precompiled-source.js');
     await writeFile(precompiledSourcePath, precompiledSource);
+    if (precompiledSourceMap) {
+      await writeFile(precompiledSourcePath + '.map', JSON.stringify(precompiledSourceMap, null, 2));
+    }
 
     const { component } = await componentize({
       sourcePath: precompiledSourcePath,

packages/build-tools/src/index.ts

@@ -17,7 +17,7 @@ const POSTAMBLE = '}';
 /// will intern regular expressions, duplicating them at the top level and testing them with both
 /// an ascii and utf8 string should ensure that they won't be re-compiled when run in the fetch
 /// handler.
-export function precompile(source, filename = '<input>', moduleMode = false) {
+export function precompile(source, filename = '<input>', moduleMode = false, precompiledFileName = 'precompiled-source.js') {
     const magicString = new MagicString(source, {
         filename,
     });
@@ -52,11 +52,11 @@ export function precompile(source, filename = '<input>', moduleMode = false) {
     magicString.prepend(`${PREAMBLE}${precompileCalls.join('\n')}${POSTAMBLE}`);
 
     // When we're ready to pipe in source maps:
-    // const map = magicString.generateMap({
-    //   source: 'source.js',
-    //   file: 'converted.js.map',
-    //   includeContent: true
-    // });
+    const map = magicString.generateMap({
+        source: filename,
+        file: `${precompiledFileName}.map`,
+        includeContent: true
+    });
 
-    return magicString.toString();
+    return { content: magicString.toString(), sourceMap: map };
 }

packages/build-tools/src/precompile.js

@@ -1,9 +1,24 @@
 import { readFile } from 'fs/promises';
 import { createHash } from 'node:crypto';
 import { access, writeFile } from 'node:fs/promises';
+import remapping, { SourceMapInput } from '@ampproject/remapping';
 
+type FileName = string;
+type SourceMapLookup = Record<FileName, SourceMapInput>;
+
+export function chainSourceMaps(finalMap: SourceMapInput, sourceMapLookup: SourceMapLookup) {
+  return remapping(finalMap, (source) => {
+    const sourceMap = sourceMapLookup[source];
+    if (sourceMap) {
+      return sourceMap;
+    }
+    // If not the source, we do not want to traverse it further so we return null.
+    // This is because sometimes npm packages have their own source maps but do
+    // not have the original source files.
+    return null;
+  });
+}
 
-// Function to calculate file checksum
 export async function calculateChecksum(content: string | Buffer) {
   try {
     const hash = createHash('sha256');

packages/build-tools/src/utils.ts

@@ -7,6 +7,7 @@ const config = async () => {
         mode: 'production',
         stats: 'errors-only',
         entry: './src/index.ts',
+        devtool: 'source-map',
         experiments: {
             outputModule: true,
         },