Turbopack: fix dist dir on Windows (vercel#81758)

Regression from vercel#80683

Closes PACK-5071
Closes vercel#81628

Author: mischnic

.github/workflows/build_and_test.yml

@@ -705,10 +705,11 @@ jobs:
       afterBuild: |
         export NEXT_TEST_MODE=start
 
-        node run-tests.js \
+        node run-tests.js --type production \
           test/e2e/app-dir/app/index.test.ts \
           test/e2e/app-dir/app-edge/app-edge.test.ts \
-          test/e2e/app-dir/metadata-edge/index.test.ts
+          test/e2e/app-dir/metadata-edge/index.test.ts \
+          test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts
       stepName: 'test-prod-windows'
       runs_on_labels: '["windows","self-hosted","x64"]'
       buildNativeTarget: 'x86_64-pc-windows-msvc'

crates/napi/src/next_api/project.rs

@@ -126,15 +126,18 @@ pub struct NapiWatchOptions {
 
 #[napi(object)]
 pub struct NapiProjectOptions {
-    /// A root path from which all files must be nested under. Trying to access
-    /// a file outside this root will fail. Think of this as a chroot.
+    /// An absolute root path (Unix or Windows path) from which all files must be nested under.
+    /// Trying to access a file outside this root will fail, so think of this as a chroot.
+    /// E.g. `/home/user/projects/my-repo`.
     pub root_path: RcStr,
 
-    /// A path inside the root_path which contains the app/pages directories.
+    /// A path which contains the app/pages directories, relative to [`Project::root_path`], always
+    /// Unix path. E.g. `apps/my-app`
     pub project_path: RcStr,
 
-    /// next.config's distDir. Project initialization occurs earlier than
-    /// deserializing next.config, so passing it as separate option.
+    /// A path where to emit the build outputs, relative to [`Project::project_path`], always Unix
+    /// path. Corresponds to next.config.js's `distDir`.
+    /// E.g. `.next`
     pub dist_dir: RcStr,
 
     /// Filesystem watcher options.
@@ -180,15 +183,19 @@ pub struct NapiProjectOptions {
 /// [NapiProjectOptions] with all fields optional.
 #[napi(object)]
 pub struct NapiPartialProjectOptions {
-    /// A root path from which all files must be nested under. Trying to access
-    /// a file outside this root will fail. Think of this as a chroot.
+    /// An absolute root path  (Unix or Windows path) from which all files must be nested under.
+    /// Trying to access a file outside this root will fail, so think of this as a chroot.
+    /// E.g. `/home/user/projects/my-repo`.
     pub root_path: Option<RcStr>,
 
-    /// A path inside the root_path which contains the app/pages directories.
+    /// A path which contains the app/pages directories, relative to [`Project::root_path`], always
+    /// a Unix path.
+    /// E.g. `apps/my-app`
     pub project_path: Option<RcStr>,
 
-    /// next.config's distDir. Project initialization occurs earlier than
-    /// deserializing next.config, so passing it as separate option.
+    /// A path where to emit the build outputs, relative to [`Project::project_path`], always a
+    /// Unix path. Corresponds to next.config.js's `distDir`.
+    /// E.g. `.next`
     pub dist_dir: Option<Option<RcStr>>,
 
     /// Filesystem watcher options.
@@ -392,7 +399,9 @@ pub fn project_new(
 
             let subscriber = subscriber.with(FilterLayer::try_new(&trace).unwrap());
 
-            let internal_dir = PathBuf::from(&options.project_path).join(&options.dist_dir);
+            let internal_dir = PathBuf::from(&options.root_path)
+                .join(&options.project_path)
+                .join(&options.dist_dir);
             std::fs::create_dir_all(&internal_dir)
                 .context("Unable to create .next directory")
                 .unwrap();
@@ -1424,13 +1433,9 @@ pub async fn get_source_map_rope(
         Err(_) => (file_path.to_string(), None),
     };
 
-    let Some(chunk_base) = file.strip_prefix(
-        &(format!(
-            "{}/{}/",
-            container.project().await?.project_path,
-            container.project().dist_dir().await?
-        )),
-    ) else {
+    let Some(chunk_base) =
+        file.strip_prefix(container.project().dist_dir_absolute().await?.as_str())
+    else {
         // File doesn't exist within the dist dir
         return Ok(OptionStringifiedSourceMap::none());
     };

crates/next-api/src/project.rs

@@ -1,4 +1,4 @@
-use std::{path::MAIN_SEPARATOR, time::Duration};
+use std::time::Duration;
 
 use anyhow::{Context, Result, bail};
 use indexmap::map::Entry;
@@ -37,8 +37,8 @@ use turbo_tasks::{
 };
 use turbo_tasks_env::{EnvMap, ProcessEnv};
 use turbo_tasks_fs::{
-    DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, get_relative_path_to,
-    invalidation,
+    DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, invalidation,
+    util::{join_path, unix_to_sys},
 };
 use turbopack::{
     ModuleAssetContext, evaluate_context::node_build_environment,
@@ -149,11 +149,13 @@ pub struct WatchOptions {
 )]
 #[serde(rename_all = "camelCase")]
 pub struct ProjectOptions {
-    /// A root path from which all files must be nested under. Trying to access
-    /// a file outside this root will fail. Think of this as a chroot.
+    /// An absolute root path (Unix or Windows path) from which all files must be nested under.
+    /// Trying to access a file outside this root will fail, so think of this as a chroot.
+    /// E.g. `/home/user/projects/my-repo`.
     pub root_path: RcStr,
 
-    /// A path inside the root_path which contains the app/pages directories.
+    /// A path which contains the app/pages directories, relative to [`Project::root_path`], always
+    /// Unix path. E.g. `apps/my-app`
     pub project_path: RcStr,
 
     /// The contents of next.config.js, serialized to JSON.
@@ -538,15 +540,20 @@ impl ProjectContainer {
 
 #[turbo_tasks::value]
 pub struct Project {
-    /// A root path from which all files must be nested under. Trying to access
-    /// a file outside this root will fail. Think of this as a chroot.
+    /// An absolute root path (Windows or Unix path) from which all files must be nested under.
+    /// Trying to access a file outside this root will fail, so think of this as a chroot.
+    /// E.g. `/home/user/projects/my-repo`.
     root_path: RcStr,
 
-    /// A path where to emit the build outputs. next.config.js's distDir.
-    dist_dir: RcStr,
+    /// A path which contains the app/pages directories, relative to [`Project::root_path`], always
+    /// a Unix path.
+    /// E.g. `apps/my-app`
+    project_path: RcStr,
 
-    /// A path inside the root_path which contains the app/pages directories.
-    pub project_path: RcStr,
+    /// A path where to emit the build outputs, relative to [`Project::project_path`], always a
+    /// Unix path. Corresponds to next.config.js's `distDir`.
+    /// E.g. `.next`
+    dist_dir: RcStr,
 
     /// Filesystem watcher options.
     watch: WatchOptions,
@@ -685,21 +692,30 @@ impl Project {
     }
 
     #[turbo_tasks::function]
-    pub fn dist_dir(&self) -> Vc<RcStr> {
-        Vc::cell(self.dist_dir.clone())
+    pub fn dist_dir_absolute(&self) -> Result<Vc<RcStr>> {
+        Ok(Vc::cell(
+            format!(
+                "{}{}{}",
+                self.root_path,
+                std::path::MAIN_SEPARATOR,
+                unix_to_sys(
+                    &join_path(&self.project_path, &self.dist_dir)
+                        .context("expected project_path to be inside of root_path")?
+                )
+            )
+            .into(),
+        ))
     }
 
     #[turbo_tasks::function]
     pub async fn node_root(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
         let this = self.await?;
-        let relative_from_root_to_project_path =
-            get_relative_path_to(&this.root_path, &this.project_path);
         Ok(self
             .output_fs()
             .root()
             .await?
-            .join(&relative_from_root_to_project_path)?
-            .join(&this.dist_dir.clone())?
+            .join(&this.project_path)?
+            .join(&this.dist_dir)?
             .cell())
     }
 
@@ -726,28 +742,24 @@ impl Project {
             .cell())
     }
 
+    /// Returns the relative path from the node root to the output root.
+    /// E.g. from `[project]/test/e2e/app-dir/non-root-project-monorepo/apps/web/app/
+    /// import-meta-url-ssr/page.tsx` to `[project]/`.
     #[turbo_tasks::function]
     pub async fn node_root_to_root_path(self: Vc<Self>) -> Result<Vc<RcStr>> {
-        let this = self.await?;
-        let output_root_to_root_path = self
-            .project_path()
-            .await?
-            .join(&this.dist_dir.clone())?
-            .get_relative_path_to(&*self.project_root_path().await?)
-            .context("Project path need to be in root path")?;
-        Ok(Vc::cell(output_root_to_root_path))
+        Ok(Vc::cell(
+            self.node_root()
+                .await?
+                .get_relative_path_to(&*self.output_fs().root().await?)
+                .context("Expected node root to be inside of output fs")?,
+        ))
     }
 
     #[turbo_tasks::function]
     pub async fn project_path(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
         let this = self.await?;
         let root = self.project_root_path().await?;
-        let project_relative = this.project_path.strip_prefix(&*this.root_path).unwrap();
-        let project_relative = project_relative
-            .strip_prefix(MAIN_SEPARATOR)
-            .unwrap_or(project_relative)
-            .replace(MAIN_SEPARATOR, "/");
-        Ok(root.join(&project_relative)?.cell())
+        Ok(root.join(&this.project_path)?.cell())
     }
 
     #[turbo_tasks::function]

packages/next/src/build/swc/generated-native.d.ts

@@ -100,15 +100,20 @@ export interface NapiWatchOptions {
 }
 export interface NapiProjectOptions {
   /**
-   * A root path from which all files must be nested under. Trying to access
-   * a file outside this root will fail. Think of this as a chroot.
+   * An absolute root path from which all files must be nested under. Trying to access
+   * a file outside this root will fail, so think of this as a chroot.
+   * E.g. `/home/user/projects/my-repo`.
    */
   rootPath: RcStr
-  /** A path inside the root_path which contains the app/pages directories. */
+  /**
+   * A path which contains the app/pages directories, relative to [`Project::root_path`].
+   * E.g. `apps/my-app`
+   */
   projectPath: RcStr
   /**
-   * next.config's distDir. Project initialization occurs earlier than
-   * deserializing next.config, so passing it as separate option.
+   * A path where to emit the build outputs, relative to [`Project::project_path`].
+   * Corresponds to next.config.js's `distDir`.
+   * E.g. `.next`
    */
   distDir: RcStr
   /** Filesystem watcher options. */
@@ -146,15 +151,20 @@ export interface NapiProjectOptions {
 /** [NapiProjectOptions] with all fields optional. */
 export interface NapiPartialProjectOptions {
   /**
-   * A root path from which all files must be nested under. Trying to access
-   * a file outside this root will fail. Think of this as a chroot.
+   * An absolute root path from which all files must be nested under. Trying to access
+   * a file outside this root will fail, so think of this as a chroot.
+   * E.g. `/home/user/projects/my-repo`.
    */
   rootPath?: RcStr
-  /** A path inside the root_path which contains the app/pages directories. */
+  /**
+   * A path which contains the app/pages directories, relative to [`Project::root_path`].
+   * E.g. `apps/my-app`
+   */
   projectPath?: RcStr
   /**
-   * next.config's distDir. Project initialization occurs earlier than
-   * deserializing next.config, so passing it as separate option.
+   * A path where to emit the build outputs, relative to [`Project::project_path`].
+   * Corresponds to next.config.js's `distDir`.
+   * E.g. `.next`
    */
   distDir?: RcStr | undefined | null
   /** Filesystem watcher options. */

packages/next/src/build/swc/index.ts

@@ -621,7 +621,7 @@ function bindingToApi(
       ...options,
       nextConfig: await serializeNextConfig(
         options.nextConfig,
-        options.projectPath!
+        path.join(options.rootPath, options.projectPath)
       ),
       jsConfig: JSON.stringify(options.jsConfig),
       env: rustifyEnv(options.env),
@@ -635,7 +635,10 @@ function bindingToApi(
       ...options,
       nextConfig:
         options.nextConfig &&
-        (await serializeNextConfig(options.nextConfig, options.projectPath!)),
+        (await serializeNextConfig(
+          options.nextConfig,
+          path.join(options.rootPath!, options.projectPath!)
+        )),
       jsConfig: options.jsConfig && JSON.stringify(options.jsConfig),
       env: options.env && rustifyEnv(options.env),
     }

packages/next/src/build/swc/types.ts

@@ -346,18 +346,22 @@ export type WrittenEndpoint =
 
 export interface ProjectOptions {
   /**
-   * A root path from which all files must be nested under. Trying to access
-   * a file outside this root will fail. Think of this as a chroot.
+   * An absolute root path (Unix or Windows path) from which all files must be nested under. Trying
+   * to access a file outside this root will fail, so think of this as a chroot.
+   * E.g. `/home/user/projects/my-repo`.
    */
   rootPath: string
 
   /**
-   * A path inside the root_path which contains the app/pages directories.
+   * A path which contains the app/pages directories, relative to `root_path`, always a Unix path.
+   * E.g. `apps/my-app`
    */
   projectPath: string
 
   /**
-   * The path to the .next directory.
+   * A path where to emit the build outputs, relative to [`Project::project_path`], always a Unix
+   * path. Corresponds to next.config.js's `distDir`.
+   * E.g. `.next`
    */
   distDir: string
 

packages/next/src/build/turbopack-build/impl.ts

@@ -22,6 +22,7 @@ import { setGlobal } from '../../trace'
 import { isCI } from '../../server/ci-info'
 import { backgroundLogCompilationEvents } from '../../shared/lib/turbopack/compilation-events'
 import { getSupportedBrowsers } from '../utils'
+import { normalizePath } from '../../lib/normalize-path'
 
 export async function turbopackBuild(): Promise<{
   duration: number
@@ -52,10 +53,11 @@ export async function turbopackBuild(): Promise<{
   const supportedBrowsers = getSupportedBrowsers(dir, dev)
 
   const persistentCaching = isPersistentCachingEnabled(config)
+  const rootPath = config.turbopack?.root || config.outputFileTracingRoot || dir
   const project = await bindings.turbo.createProject(
     {
-      projectPath: dir,
       rootPath: config.turbopack?.root || config.outputFileTracingRoot || dir,
+      projectPath: normalizePath(path.relative(rootPath, dir) || '.'),
       distDir,
       nextConfig: config,
       jsConfig: await getTurbopackJsConfig(dir, config),

packages/next/src/build/webpack/loaders/css-loader/src/utils.ts

@@ -11,6 +11,7 @@ import localByDefault from 'next/dist/compiled/postcss-modules-local-by-default'
 import extractImports from 'next/dist/compiled/postcss-modules-extract-imports'
 import modulesScope from 'next/dist/compiled/postcss-modules-scope'
 import camelCase from './camelcase'
+import { normalizePath } from '../../../../../lib/normalize-path'
 
 const whitespace = '[\\x20\\t\\r\\n\\f]'
 const unescapeRegExp = new RegExp(
@@ -39,10 +40,6 @@ function unescape(str: string) {
   })
 }
 
-function normalizePath(file: string) {
-  return path.sep === '\\' ? file.replace(/\\/g, '/') : file
-}
-
 function fixedEncodeURIComponent(str: string) {
   return str.replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16)}`)
 }

packages/next/src/lib/normalize-path.ts

@@ -0,0 +1,5 @@
+import path from 'path'
+
+export function normalizePath(file: string) {
+  return path.sep === '\\' ? file.replace(/\\/g, '/') : file
+}