Initial MCP implementation (#81770)

### What?

Add MCP for next dev exposing entrypoints, module graph and issues.

Can be enabled via `NEXT_EXPERIMENTAL_MCP_SECRET=<random>` and makes the MCP available on `/_next/mcp?<random>`.

Interesting change is in: packages/next/src/server/lib/router-utils/mcp.ts

Author: sokra

crates/napi/src/next_api/endpoint.rs

@@ -4,21 +4,25 @@ use anyhow::Result;
 use futures_util::TryFutureExt;
 use napi::{JsFunction, bindgen_prelude::External};
 use next_api::{
+    module_graph_snapshot::{ModuleGraphSnapshot, get_module_graph_snapshot},
     operation::OptionEndpoint,
     paths::ServerPath,
     route::{
-        EndpointOutputPaths, endpoint_client_changed_operation, endpoint_server_changed_operation,
-        endpoint_write_to_disk_operation,
+        Endpoint, EndpointOutputPaths, endpoint_client_changed_operation,
+        endpoint_server_changed_operation, endpoint_write_to_disk_operation,
     },
 };
 use tracing::Instrument;
-use turbo_tasks::{Completion, Effects, OperationVc, ReadRef, Vc};
-use turbopack_core::{diagnostics::PlainDiagnostic, issue::PlainIssue};
+use turbo_tasks::{
+    Completion, Effects, OperationVc, ReadRef, TryFlatJoinIterExt, TryJoinIterExt, Vc,
+};
+use turbopack_core::{diagnostics::PlainDiagnostic, error::PrettyPrintError, issue::PlainIssue};
 
 use super::utils::{
     DetachedVc, NapiDiagnostic, NapiIssue, RootTask, TurbopackResult,
     strongly_consistent_catch_collectables, subscribe,
 };
+use crate::next_api::module_graph::NapiModuleGraphSnapshot;
 
 #[napi(object)]
 #[derive(Default)]
@@ -81,6 +85,11 @@ impl From<Option<EndpointOutputPaths>> for NapiWrittenEndpoint {
     }
 }
 
+#[napi(object)]
+pub struct NapiModuleGraphSnapshots {
+    pub module_graphs: Vec<NapiModuleGraphSnapshot>,
+}
+
 // NOTE(alexkirsz) We go through an extra layer of indirection here because of
 // two factors:
 // 1. rustc currently has a bug where using a dyn trait as a type argument to
@@ -155,6 +164,105 @@ pub async fn endpoint_write_to_disk(
     })
 }
 
+#[turbo_tasks::value(serialization = "none")]
+struct ModuleGraphsWithIssues {
+    module_graphs: Option<ReadRef<ModuleGraphSnapshots>>,
+    issues: Arc<Vec<ReadRef<PlainIssue>>>,
+    diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
+    effects: Arc<Effects>,
+}
+
+#[turbo_tasks::function(operation)]
+async fn get_module_graphs_with_issues_operation(
+    endpoint_op: OperationVc<OptionEndpoint>,
+) -> Result<Vc<ModuleGraphsWithIssues>> {
+    let module_graphs_op = get_module_graphs_operation(endpoint_op);
+    let (module_graphs, issues, diagnostics, effects) =
+        strongly_consistent_catch_collectables(module_graphs_op).await?;
+    Ok(ModuleGraphsWithIssues {
+        module_graphs,
+        issues,
+        diagnostics,
+        effects,
+    }
+    .cell())
+}
+
+#[turbo_tasks::value(transparent)]
+struct ModuleGraphSnapshots(Vec<ReadRef<ModuleGraphSnapshot>>);
+
+#[turbo_tasks::function(operation)]
+async fn get_module_graphs_operation(
+    endpoint_op: OperationVc<OptionEndpoint>,
+) -> Result<Vc<ModuleGraphSnapshots>> {
+    let Some(endpoint) = *endpoint_op.connect().await? else {
+        return Ok(Vc::cell(vec![]));
+    };
+    let graphs = endpoint.module_graphs().await?;
+    let entries = endpoint.entries().await?;
+    let entry_modules = entries.iter().flat_map(|e| e.entries()).collect::<Vec<_>>();
+    let snapshots = graphs
+        .iter()
+        .map(async |&graph| {
+            let module_graph = graph.await?;
+            let entry_modules = entry_modules
+                .iter()
+                .map(async |&m| Ok(module_graph.has_entry(m).await?.then_some(m)))
+                .try_flat_join()
+                .await?;
+            Ok((*graph, entry_modules))
+        })
+        .try_join()
+        .await?
+        .into_iter()
+        .map(|(graph, entry_modules)| (graph, Vc::cell(entry_modules)))
+        .collect::<Vec<_>>()
+        .into_iter()
+        .map(async |(graph, entry_modules)| {
+            get_module_graph_snapshot(graph, Some(entry_modules)).await
+        })
+        .try_join()
+        .await?;
+    Ok(Vc::cell(snapshots))
+}
+
+#[napi]
+pub async fn endpoint_module_graphs(
+    #[napi(ts_arg_type = "{ __napiType: \"Endpoint\" }")] endpoint: External<ExternalEndpoint>,
+) -> napi::Result<TurbopackResult<NapiModuleGraphSnapshots>> {
+    let endpoint_op: OperationVc<OptionEndpoint> = ***endpoint;
+    let (module_graphs, issues, diagnostics) = endpoint
+        .turbopack_ctx()
+        .turbo_tasks()
+        .run_once(async move {
+            let module_graphs_op = get_module_graphs_with_issues_operation(endpoint_op);
+            let ModuleGraphsWithIssues {
+                module_graphs,
+                issues,
+                diagnostics,
+                effects: _,
+            } = &*module_graphs_op.connect().await?;
+            Ok((module_graphs.clone(), issues.clone(), diagnostics.clone()))
+        })
+        .await
+        .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
+
+    Ok(TurbopackResult {
+        result: NapiModuleGraphSnapshots {
+            module_graphs: module_graphs
+                .into_iter()
+                .flat_map(|m| m.into_iter())
+                .map(|m| NapiModuleGraphSnapshot::from(&**m))
+                .collect(),
+        },
+        issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
+        diagnostics: diagnostics
+            .iter()
+            .map(|d| NapiDiagnostic::from(d))
+            .collect(),
+    })
+}
+
 #[napi(ts_return_type = "{ __napiType: \"RootTask\" }")]
 pub fn endpoint_server_changed_subscribe(
     #[napi(ts_arg_type = "{ __napiType: \"Endpoint\" }")] endpoint: External<ExternalEndpoint>,

crates/napi/src/next_api/mod.rs

@@ -1,4 +1,5 @@
 pub mod endpoint;
+pub mod module_graph;
 pub mod project;
 pub mod turbopack_ctx;
 pub mod utils;

crates/napi/src/next_api/module_graph.rs

@@ -0,0 +1,98 @@
+use next_api::module_graph_snapshot::{ModuleGraphSnapshot, ModuleInfo, ModuleReference};
+use turbo_rcstr::RcStr;
+use turbopack_core::chunk::ChunkingType;
+
+#[napi(object)]
+pub struct NapiModuleReference {
+    /// The index of the referenced/referencing module in the modules list.
+    pub index: u32,
+    /// The export used in the module reference.
+    pub export: String,
+    /// The type of chunking for the module reference.
+    pub chunking_type: String,
+}
+
+impl From<&ModuleReference> for NapiModuleReference {
+    fn from(reference: &ModuleReference) -> Self {
+        Self {
+            index: reference.index as u32,
+            export: reference.export.to_string(),
+            chunking_type: match &reference.chunking_type {
+                ChunkingType::Parallel { hoisted: true, .. } => "hoisted".to_string(),
+                ChunkingType::Parallel { hoisted: false, .. } => "sync".to_string(),
+                ChunkingType::Async => "async".to_string(),
+                ChunkingType::Isolated {
+                    merge_tag: None, ..
+                } => "isolated".to_string(),
+                ChunkingType::Isolated {
+                    merge_tag: Some(name),
+                    ..
+                } => format!("isolated {name}"),
+                ChunkingType::Shared {
+                    merge_tag: None, ..
+                } => "shared".to_string(),
+                ChunkingType::Shared {
+                    merge_tag: Some(name),
+                    ..
+                } => format!("shared {name}"),
+                ChunkingType::Traced => "traced".to_string(),
+            },
+        }
+    }
+}
+
+#[napi(object)]
+pub struct NapiModuleInfo {
+    pub ident: RcStr,
+    pub path: RcStr,
+    pub depth: u32,
+    pub size: u32,
+    pub retained_size: u32,
+    pub references: Vec<NapiModuleReference>,
+    pub incoming_references: Vec<NapiModuleReference>,
+}
+
+impl From<&ModuleInfo> for NapiModuleInfo {
+    fn from(info: &ModuleInfo) -> Self {
+        Self {
+            ident: info.ident.clone(),
+            path: info.path.clone(),
+            depth: info.depth,
+            size: info.size,
+            retained_size: info.retained_size,
+            references: info
+                .references
+                .iter()
+                .map(NapiModuleReference::from)
+                .collect(),
+            incoming_references: info
+                .incoming_references
+                .iter()
+                .map(NapiModuleReference::from)
+                .collect(),
+        }
+    }
+}
+
+#[napi(object)]
+#[derive(Default)]
+pub struct NapiModuleGraphSnapshot {
+    pub modules: Vec<NapiModuleInfo>,
+    pub entries: Vec<u32>,
+}
+
+impl From<&ModuleGraphSnapshot> for NapiModuleGraphSnapshot {
+    fn from(snapshot: &ModuleGraphSnapshot) -> Self {
+        Self {
+            modules: snapshot.modules.iter().map(NapiModuleInfo::from).collect(),
+            entries: snapshot
+                .entries
+                .iter()
+                .map(|&i| {
+                    // If you have more that 4294967295 entries, you probably have other problems...
+                    i.try_into().unwrap()
+                })
+                .collect(),
+        }
+    }
+}

crates/napi/src/next_api/project.rs

@@ -9,6 +9,7 @@ use napi::{
 };
 use next_api::{
     entrypoints::Entrypoints,
+    module_graph_snapshot::{ModuleGraphSnapshot, get_module_graph_snapshot},
     operation::{
         EntrypointsOperation, InstrumentationOperation, MiddlewareOperation, OptionEndpoint,
         RouteOperation,
@@ -63,13 +64,14 @@ use url::Url;
 use crate::{
     next_api::{
         endpoint::ExternalEndpoint,
+        module_graph::NapiModuleGraphSnapshot,
         turbopack_ctx::{
             NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject, NextTurboTasks,
             NextTurbopackContext, create_turbo_tasks,
         },
         utils::{
             DetachedVc, NapiDiagnostic, NapiIssue, RootTask, TurbopackResult, get_diagnostics,
-            get_issues, subscribe,
+            get_issues, strongly_consistent_catch_collectables, subscribe,
         },
     },
     register,
@@ -987,6 +989,40 @@ async fn output_assets_operation(
     Ok(Vc::cell(output_assets.into_iter().collect()))
 }
 
+#[napi]
+pub async fn project_entrypoints(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+) -> napi::Result<TurbopackResult<NapiEntrypoints>> {
+    let container = project.container;
+
+    let (entrypoints, issues, diags) = project
+        .turbopack_ctx
+        .turbo_tasks()
+        .run_once(async move {
+            let entrypoints_with_issues_op = get_entrypoints_with_issues_operation(container);
+
+            // Read and compile the files
+            let EntrypointsWithIssues {
+                entrypoints,
+                issues,
+                diagnostics,
+                effects: _,
+            } = &*entrypoints_with_issues_op
+                .read_strongly_consistent()
+                .await?;
+
+            Ok((entrypoints.clone(), issues.clone(), diagnostics.clone()))
+        })
+        .await
+        .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
+
+    Ok(TurbopackResult {
+        result: NapiEntrypoints::from_entrypoints_op(&entrypoints, &project.turbopack_ctx)?,
+        issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
+        diagnostics: diags.iter().map(|d| NapiDiagnostic::from(d)).collect(),
+    })
+}
+
 #[napi(ts_return_type = "{ __napiType: \"RootTask\" }")]
 pub fn project_entrypoints_subscribe(
     #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
@@ -1650,3 +1686,70 @@ pub fn project_get_source_map_sync(
         tokio::runtime::Handle::current().block_on(project_get_source_map(project, file_path))
     })
 }
+
+#[napi]
+pub async fn project_module_graph(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+) -> napi::Result<TurbopackResult<NapiModuleGraphSnapshot>> {
+    let container = project.container;
+    let (module_graph, issues, diagnostics) = project
+        .turbopack_ctx
+        .turbo_tasks()
+        .run_once(async move {
+            let module_graph_op = get_module_graph_with_issues_operation(container);
+            let ModuleGraphWithIssues {
+                module_graph,
+                issues,
+                diagnostics,
+                effects: _,
+            } = &*module_graph_op.connect().await?;
+            Ok((module_graph.clone(), issues.clone(), diagnostics.clone()))
+        })
+        .await
+        .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
+
+    Ok(TurbopackResult {
+        result: module_graph.map_or_else(NapiModuleGraphSnapshot::default, |m| {
+            NapiModuleGraphSnapshot::from(&*m)
+        }),
+        issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
+        diagnostics: diagnostics
+            .iter()
+            .map(|d| NapiDiagnostic::from(d))
+            .collect(),
+    })
+}
+
+#[turbo_tasks::value(serialization = "none")]
+struct ModuleGraphWithIssues {
+    module_graph: Option<ReadRef<ModuleGraphSnapshot>>,
+    issues: Arc<Vec<ReadRef<PlainIssue>>>,
+    diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
+    effects: Arc<Effects>,
+}
+
+#[turbo_tasks::function(operation)]
+async fn get_module_graph_with_issues_operation(
+    project: ResolvedVc<ProjectContainer>,
+) -> Result<Vc<ModuleGraphWithIssues>> {
+    let module_graph_op = get_module_graph_operation(project);
+    let (module_graph, issues, diagnostics, effects) =
+        strongly_consistent_catch_collectables(module_graph_op).await?;
+    Ok(ModuleGraphWithIssues {
+        module_graph,
+        issues,
+        diagnostics,
+        effects,
+    }
+    .cell())
+}
+
+#[turbo_tasks::function(operation)]
+async fn get_module_graph_operation(
+    project: ResolvedVc<ProjectContainer>,
+) -> Result<Vc<ModuleGraphSnapshot>> {
+    let project = project.project();
+    let graph = project.whole_app_module_graphs().await?.full;
+    let snapshot = get_module_graph_snapshot(*graph, None).resolve().await?;
+    Ok(snapshot)
+}