Native node graph runtime

Author: adamgerhant

Cargo.lock

@@ -150,7 +150,10 @@ impl Dispatcher {
 				Message::Frontend(message) => {
 					// Handle these messages immediately by returning early
 					if let FrontendMessage::TriggerFontLoad { .. } = message {
-						self.responses.push(message);
+						// Deduplicate the render native node graph messages. TODO: Replace responses with hashset
+						if !(message == FrontendMessage::RequestNativeNodeGraphRender && self.responses.contains(&FrontendMessage::RequestNativeNodeGraphRender)) {
+							self.responses.push(message);
+						}
 						self.cleanup_queues(false);
 
 						// Return early to avoid running the code after the match block

editor/src/dispatcher.rs

@@ -267,6 +267,7 @@ pub enum FrontendMessage {
 	UpdateMouseCursor {
 		cursor: MouseCursorIcon,
 	},
+	RequestNativeNodeGraphRender,
 	UpdateNodeGraphSvelteRender {
 		#[serde(rename = "nodesToRender")]
 		nodes_to_render: Vec<FrontendNodeToRender>,

editor/src/messages/frontend/frontend_message.rs

@@ -1626,6 +1626,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphMessageContext<'a>> for NodeG
 				responses.add(NodeGraphMessage::UpdateActionButtons);
 
 				if self.native_node_graph_render {
+					responses.add(FrontendMessage::RequestNativeNodeGraphRender);
 				} else {
 					let nodes_to_render = network_interface.collect_nodes(&self.node_graph_errors, preferences.graph_wire_style, breadcrumb_network_path);
 					self.frontend_nodes = nodes_to_render.iter().map(|node| node.metadata.node_id).collect();

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs

@@ -25,7 +25,9 @@ editor = { path = "../../editor", package = "graphite-editor", features = [
 	"resvg",
 	"vello",
 ] }
+interpreted-executor = { workspace = true }
 graphene-std = { workspace = true }
+once_cell = { workspace = true }
 
 # Workspace dependencies
 graph-craft = { workspace = true }

frontend/wasm/Cargo.toml

@@ -5,7 +5,9 @@
 // on the dispatcher messaging system and more complex Rust data types.
 //
 use crate::helpers::translate_key;
-use crate::{EDITOR_HANDLE, EDITOR_HAS_CRASHED, Error, MESSAGE_BUFFER};
+#[cfg(not(feature = "native"))]
+use crate::wasm_node_graph_ui_executor::WasmNodeGraphUIExecutor;
+use crate::{EDITOR_HANDLE, EDITOR_HAS_CRASHED, Error, MESSAGE_BUFFER, WASM_NODE_GRAPH_EXECUTOR};
 use editor::consts::FILE_EXTENSION;
 use editor::messages::input_mapper::utility_types::input_keyboard::ModifierKeys;
 use editor::messages::input_mapper::utility_types::input_mouse::{EditorMouseState, ScrollDelta, ViewportBounds};
@@ -17,6 +19,7 @@ use editor::messages::tool::tool_messages::tool_prelude::WidgetId;
 use graph_craft::document::NodeId;
 use graphene_std::raster::Image;
 use graphene_std::raster::color::Color;
+use interpreted_executor::ui_runtime::CompilationRequest;
 use js_sys::{Object, Reflect};
 use serde::Serialize;
 use serde_wasm_bindgen::{self, from_value};
@@ -149,9 +152,13 @@ impl EditorHandle {
 	pub fn new(frontend_message_handler_callback: js_sys::Function) -> Self {
 		let editor = Editor::new();
 		let editor_handle = EditorHandle { frontend_message_handler_callback };
+		let node_graph_executor = WasmNodeGraphUIExecutor::new();
 		if EDITOR.with(|handle| handle.lock().ok().map(|mut guard| *guard = Some(editor))).is_none() {
 			log::error!("Attempted to initialize the editor more than once");
 		}
+		if WASM_NODE_GRAPH_EXECUTOR.with(|handle| handle.lock().ok().map(|mut guard| *guard = Some(node_graph_executor))).is_none() {
+			log::error!("Attempted to initialize the editor more than once");
+		}
 		if EDITOR_HANDLE.with(|handle| handle.lock().ok().map(|mut guard| *guard = Some(editor_handle.clone()))).is_none() {
 			log::error!("Attempted to initialize the editor handle more than once");
 		}
@@ -208,6 +215,17 @@ impl EditorHandle {
 
 	// Sends a FrontendMessage to JavaScript
 	fn send_frontend_message_to_js(&self, mut message: FrontendMessage) {
+		// Intercept any requests to render the node graph overlay
+		if message == FrontendMessage::RequestNativeNodeGraphRender {
+			executor_editor_and_handle(|executor, editor, _handle| {
+				if let Some(node_graph_overlay_network) = editor.generate_node_graph_overlay_network() {
+					let compilation_request = CompilationRequest { network: node_graph_overlay_network };
+					executor.compilation_request(compilation_request);
+				}
+			});
+			return;
+		}
+
 		if let FrontendMessage::UpdateImageData { ref image_data } = message {
 			let new_hash = calculate_hash(image_data);
 			let prev_hash = IMAGE_DATA_HASH.load(Ordering::Relaxed);
@@ -264,6 +282,18 @@ impl EditorHandle {
 				#[cfg(not(feature = "native"))]
 				wasm_bindgen_futures::spawn_local(poll_node_graph_evaluation());
 
+				// Poll the UI node graph
+				#[cfg(not(feature = "native"))]
+				executor_editor_and_handle(|executor, editor, handle| {
+					for frontend_message in executor
+						.poll_node_graph_ui_evaluation(editor)
+						.into_iter()
+						.flat_map(|runtime_response| editor.handle_message(runtime_response))
+					{
+						handle.send_frontend_message_to_js(frontend_message);
+					}
+				});
+
 				if !EDITOR_HAS_CRASHED.load(Ordering::SeqCst) {
 					handle(|handle| {
 						// Process all messages that have been queued up
@@ -989,6 +1019,31 @@ fn editor<T: Default>(callback: impl FnOnce(&mut editor::application::Editor) ->
 	})
 }
 
+#[cfg(not(feature = "native"))]
+fn executor<T: Default>(callback: impl FnOnce(&mut WasmNodeGraphUIExecutor) -> T) -> T {
+	WASM_NODE_GRAPH_EXECUTOR.with(|executor| {
+		let mut guard = executor.try_lock();
+		let Ok(Some(executor)) = guard.as_deref_mut() else {
+			log::error!("Failed to borrow editor");
+			return T::default();
+		};
+
+		callback(executor)
+	})
+}
+
+#[cfg(not(feature = "native"))]
+pub(crate) fn executor_editor_and_handle(callback: impl FnOnce(&mut WasmNodeGraphUIExecutor, &mut Editor, &mut EditorHandle)) {
+	executor(|executor| {
+		handle(|editor_handle| {
+			editor(|editor| {
+				// Call the closure with the editor and its handle
+				callback(executor, editor, editor_handle);
+			})
+		});
+	})
+}
+
 /// Provides access to the `Editor` and its `EditorHandle` by calling the given closure with them as arguments.
 #[cfg(not(feature = "native"))]
 pub(crate) fn editor_and_handle(callback: impl FnOnce(&mut Editor, &mut EditorHandle)) {
@@ -1046,7 +1101,6 @@ async fn poll_node_graph_evaluation() {
 		// If the editor cannot be borrowed then it has encountered a panic - we should just ignore new dispatches
 	});
 }
-
 fn auto_save_all_documents() {
 	// Process no further messages after a crash to avoid spamming the console
 	if EDITOR_HAS_CRASHED.load(Ordering::SeqCst) {

frontend/wasm/src/editor_api.rs

@@ -7,13 +7,16 @@ extern crate log;
 pub mod editor_api;
 pub mod helpers;
 pub mod native_communcation;
+pub mod wasm_node_graph_ui_executor;
 
 use editor::messages::prelude::*;
 use std::panic;
 use std::sync::Mutex;
 use std::sync::atomic::{AtomicBool, Ordering};
 use wasm_bindgen::prelude::*;
 
+use crate::wasm_node_graph_ui_executor::WasmNodeGraphUIExecutor;
+
 // Set up the persistent editor backend state
 pub static EDITOR_HAS_CRASHED: AtomicBool = AtomicBool::new(false);
 pub static NODE_GRAPH_ERROR_DISPLAYED: AtomicBool = AtomicBool::new(false);
@@ -22,6 +25,8 @@ pub static LOGGER: WasmLog = WasmLog;
 thread_local! {
 	#[cfg(not(feature = "native"))]
 	pub static EDITOR: Mutex<Option<editor::application::Editor>> = const { Mutex::new(None) };
+	#[cfg(not(feature = "native"))]
+	pub static WASM_NODE_GRAPH_EXECUTOR: Mutex<Option<WasmNodeGraphUIExecutor>> = const { Mutex::new(None) };
 	pub static MESSAGE_BUFFER: std::cell::RefCell<Vec<Message>> = const { std::cell::RefCell::new(Vec::new()) };
 	pub static EDITOR_HANDLE: Mutex<Option<editor_api::EditorHandle>> = const { Mutex::new(None) };
 }

frontend/wasm/src/lib.rs

@@ -0,0 +1,134 @@
+use std::sync::{Mutex, mpsc::Receiver};
+
+use editor::{
+	application::Editor,
+	messages::prelude::{FrontendMessage, Message},
+};
+use graph_craft::graphene_compiler::Compiler;
+use graphene_std::node_graph_overlay::{types::NodeGraphTransform, ui_context::UIRuntimeResponse};
+use interpreted_executor::{
+	dynamic_executor::DynamicExecutor,
+	ui_runtime::{CompilationRequest, EvaluationRequest, NodeGraphUIRuntime},
+};
+use once_cell::sync::Lazy;
+
+pub static NODE_UI_RUNTIME: Lazy<Mutex<Option<NodeGraphUIRuntime>>> = Lazy::new(|| Mutex::new(None));
+
+// Since the runtime is not locked, it is possible to spawn multiple futures concurrently.
+// This is why the runtime_busy flag exists
+// This struct should never be locked in a future
+pub struct WasmNodeGraphUIExecutor {
+	response_receiver: Receiver<UIRuntimeResponse>,
+	runtime_busy: bool,
+	queued_compilation: Option<CompilationRequest>,
+}
+
+impl Default for WasmNodeGraphUIExecutor {
+	fn default() -> Self {
+		Self::new()
+	}
+}
+
+impl WasmNodeGraphUIExecutor {
+	pub fn new() -> Self {
+		let (response_sender, response_receiver) = std::sync::mpsc::channel();
+		let runtime = NodeGraphUIRuntime {
+			executor: DynamicExecutor::default(),
+			compiler: Compiler {},
+			response_sender,
+		};
+		if let Ok(mut node_runtime) = NODE_UI_RUNTIME.lock() {
+			node_runtime.replace(runtime);
+		} else {
+			log::error!("Could not lock runtime when creating new executor");
+		};
+
+		WasmNodeGraphUIExecutor {
+			response_receiver,
+			runtime_busy: false,
+			queued_compilation: None,
+		}
+	}
+
+	pub fn compilation_request(&mut self, compilation_request: CompilationRequest) {
+		if !self.runtime_busy {
+			self.runtime_busy = true;
+			wasm_bindgen_futures::spawn_local(async move {
+				let Ok(mut runtime) = NODE_UI_RUNTIME.try_lock() else {
+					log::error!("Could not get runtime when evaluating");
+					return;
+				};
+				let Some(runtime) = runtime.as_mut() else {
+					log::error!("Could not lock runtime when evaluating");
+					return;
+				};
+				runtime.compile(compilation_request).await;
+			})
+		} else {
+			self.queued_compilation = Some(compilation_request);
+		}
+	}
+
+	// Evaluates the node graph in a spawned future, and returns responses with the response_sender
+	fn evaluation_request(&mut self, editor: &Editor) {
+		if let Some(active_document) = editor.dispatcher.message_handlers.portfolio_message_handler.active_document() {
+			let Some(network_metadata) = active_document.network_interface.network_metadata(&active_document.breadcrumb_network_path) else {
+				return;
+			};
+
+			let transform = active_document.navigation_handler.calculate_offset_transform(
+				editor.dispatcher.message_handlers.input_preprocessor_message_handler.viewport_bounds.center(),
+				&network_metadata.persistent_metadata.navigation_metadata.node_graph_ptz,
+			);
+
+			let transform = NodeGraphTransform {
+				scale: transform.matrix2.x_axis.x,
+				x: transform.translation.x,
+				y: transform.translation.y,
+			};
+			let resolution = editor.dispatcher.message_handlers.input_preprocessor_message_handler.viewport_bounds.size().as_uvec2();
+			let evaluation_request = EvaluationRequest { transform, resolution };
+			self.runtime_busy = true;
+
+			wasm_bindgen_futures::spawn_local(async move {
+				let Ok(mut runtime) = NODE_UI_RUNTIME.try_lock() else {
+					log::error!("Could not get runtime when evaluating");
+					return;
+				};
+				let Some(runtime) = runtime.as_mut() else {
+					log::error!("Could not lock runtime when evaluating");
+					return;
+				};
+				runtime.evaluate(evaluation_request).await
+			})
+		}
+	}
+
+	// This is run every time a frame is requested to be rendered.
+	// It returns back Messages for how to update the frontend/editor click targets
+	// It also checks for any queued evaluation/compilation requests and runs them
+	pub fn poll_node_graph_ui_evaluation(&mut self, editor: &Editor) -> Vec<Message> {
+		let mut responses = Vec::new();
+		for runtime_response in self.response_receiver.try_iter() {
+			match runtime_response {
+				UIRuntimeResponse::RuntimeReady => {
+					self.runtime_busy = false;
+				}
+				UIRuntimeResponse::OverlaySVG(svg_string) => {
+					responses.push(FrontendMessage::UpdateNativeNodeGraphSVG { svg_string }.into());
+				}
+				UIRuntimeResponse::OverlayTexture(_texture) => todo!(),
+			}
+		}
+
+		if !self.runtime_busy {
+			if let Some(compilation_request) = self.queued_compilation.take() {
+				self.compilation_request(compilation_request);
+			} else {
+				self.evaluation_request(editor);
+			}
+		}
+
+		responses
+	}
+}

frontend/wasm/src/wasm_node_graph_ui_executor.rs

@@ -1,5 +1,6 @@
 pub mod dynamic_executor;
 pub mod node_registry;
+pub mod ui_runtime;
 pub mod util;
 
 #[cfg(test)]

node-graph/interpreted-executor/src/lib.rs

@@ -0,0 +1,58 @@
+use std::sync::{Arc, mpsc::Sender};
+
+use glam::UVec2;
+use graph_craft::{document::NodeNetwork, graphene_compiler::Compiler};
+use graphene_std::node_graph_overlay::{
+	types::NodeGraphTransform,
+	ui_context::{UIContextImpl, UIRuntimeResponse},
+};
+
+use crate::dynamic_executor::DynamicExecutor;
+
+pub struct NodeGraphUIRuntime {
+	pub executor: DynamicExecutor,
+	pub compiler: Compiler,
+	// Used within the node graph to return responses during evaluation
+	// Also used to return compilation responses, but not for the UI overlay since the types are not needed
+	pub response_sender: Sender<UIRuntimeResponse>,
+}
+
+impl NodeGraphUIRuntime {
+	pub async fn compile(&mut self, compilation_request: CompilationRequest) {
+		match self.compiler.compile_single(compilation_request.network) {
+			Ok(proto_network) => {
+				if let Err(e) = self.executor.update(proto_network).await {
+					log::error!("update error: {e:?}")
+				}
+			}
+			Err(e) => {
+				log::error!("Error compiling node graph ui network: {e:?}");
+			}
+		};
+		let _ = self.response_sender.send(UIRuntimeResponse::RuntimeReady);
+	}
+
+	pub async fn evaluate(&mut self, evaluation_request: EvaluationRequest) {
+		use graph_craft::graphene_compiler::Executor;
+
+		let ui_context = Arc::new(UIContextImpl {
+			transform: evaluation_request.transform,
+			resolution: evaluation_request.resolution,
+			response_sender: self.response_sender.clone(),
+		});
+		let _ = (&self.executor).execute(ui_context).await;
+		let _ = self.response_sender.send(UIRuntimeResponse::RuntimeReady);
+	}
+}
+
+/// Represents an update to the render state
+/// TODO: Incremental compilation
+pub struct CompilationRequest {
+	pub network: NodeNetwork,
+}
+
+// Requests an evaluation. The responses are added to the sender and can be processed when the next frame is requested
+pub struct EvaluationRequest {
+	pub transform: NodeGraphTransform,
+	pub resolution: UVec2,
+}