Render images via canvases instead of encoding and compression steps on making generating the PNG

editor/Cargo.toml

@@ -46,6 +46,7 @@ num_enum = { workspace = true }
 usvg = { workspace = true }
 once_cell = { workspace = true }
 web-sys = { workspace = true }
+bytemuck = { workspace = true }
 
 # Required dependencies
 spin = "0.9.8"

editor/src/messages/frontend/frontend_message.rs

@@ -8,6 +8,8 @@ use crate::messages::portfolio::document::utility_types::wires::{WirePath, WireP
 use crate::messages::prelude::*;
 use crate::messages::tool::utility_types::HintData;
 use graph_craft::document::NodeId;
+use graphene_std::raster::Image;
+use graphene_std::raster::TransformImage;
 use graphene_std::raster::color::Color;
 use graphene_std::text::Font;
 
@@ -179,6 +181,9 @@ pub enum FrontendMessage {
 	UpdateDocumentArtwork {
 		svg: String,
 	},
+	UpdateImageData {
+		image_data: Vec<(u64, Image<Color>, TransformImage)>,
+	},
 	UpdateDocumentBarLayout {
 		#[serde(rename = "layoutTarget")]
 		layout_target: LayoutTarget,

editor/src/messages/prelude.rs

@@ -15,13 +15,7 @@ pub use crate::messages::input_mapper::key_mapping::{KeyMappingMessage, KeyMappi
 pub use crate::messages::input_mapper::{InputMapperMessage, InputMapperMessageContext, InputMapperMessageDiscriminant, InputMapperMessageHandler};
 pub use crate::messages::input_preprocessor::{InputPreprocessorMessage, InputPreprocessorMessageContext, InputPreprocessorMessageDiscriminant, InputPreprocessorMessageHandler};
 pub use crate::messages::layout::{LayoutMessage, LayoutMessageDiscriminant, LayoutMessageHandler};
-pub use crate::messages::portfolio::document::graph_operation::{GraphOperationMessage, GraphOperationMessageContext, GraphOperationMessageDiscriminant, GraphOperationMessageHandler};
-pub use crate::messages::portfolio::document::navigation::{NavigationMessage, NavigationMessageContext, NavigationMessageDiscriminant, NavigationMessageHandler};
-pub use crate::messages::portfolio::document::node_graph::{NodeGraphMessage, NodeGraphMessageDiscriminant, NodeGraphMessageHandler};
-pub use crate::messages::portfolio::document::overlays::{OverlaysMessage, OverlaysMessageContext, OverlaysMessageDiscriminant, OverlaysMessageHandler};
-pub use crate::messages::portfolio::document::properties_panel::{PropertiesPanelMessage, PropertiesPanelMessageDiscriminant, PropertiesPanelMessageHandler};
-pub use crate::messages::portfolio::document::{DocumentMessage, DocumentMessageContext, DocumentMessageDiscriminant, DocumentMessageHandler};
-pub use crate::messages::portfolio::menu_bar::{MenuBarMessage, MenuBarMessageDiscriminant, MenuBarMessageHandler};
+pub use crate::messages::portfolio::document::graph_operation::{GraphOperationMessage, GraphOperationMessageContext, GraphOperationMessageDiscriminant, GraphOperationMessageHandler}; pub use crate::messages::portfolio::document::navigation::{NavigationMessage, NavigationMessageContext, NavigationMessageDiscriminant, NavigationMessageHandler}; pub use crate::messages::portfolio::document::node_graph::{NodeGraphMessage, NodeGraphMessageDiscriminant, NodeGraphMessageHandler}; pub use crate::messages::portfolio::document::overlays::{OverlaysMessage, OverlaysMessageContext, OverlaysMessageDiscriminant, OverlaysMessageHandler}; pub use crate::messages::portfolio::document::properties_panel::{PropertiesPanelMessage, PropertiesPanelMessageDiscriminant, PropertiesPanelMessageHandler}; pub use crate::messages::portfolio::document::{DocumentMessage, DocumentMessageContext, DocumentMessageDiscriminant, DocumentMessageHandler}; pub use crate::messages::portfolio::menu_bar::{MenuBarMessage, MenuBarMessageDiscriminant, MenuBarMessageHandler};
 pub use crate::messages::portfolio::spreadsheet::{SpreadsheetMessage, SpreadsheetMessageDiscriminant};
 pub use crate::messages::portfolio::{PortfolioMessage, PortfolioMessageContext, PortfolioMessageDiscriminant, PortfolioMessageHandler};
 pub use crate::messages::preferences::{PreferencesMessage, PreferencesMessageDiscriminant, PreferencesMessageHandler};

editor/src/node_graph_executor.rs

@@ -213,7 +213,7 @@ impl NodeGraphExecutor {
 
 	fn export(&self, node_graph_output: TaggedValue, export_config: ExportConfig, responses: &mut VecDeque<Message>) -> Result<(), String> {
 		let TaggedValue::RenderOutput(RenderOutput {
-			data: graphene_std::wasm_application_io::RenderOutputType::Svg(svg),
+			data: graphene_std::wasm_application_io::RenderOutputType::Svg { svg, .. },
 			..
 		}) = node_graph_output
 		else {
@@ -350,8 +350,9 @@ impl NodeGraphExecutor {
 		match node_graph_output {
 			TaggedValue::RenderOutput(render_output) => {
 				match render_output.data {
-					graphene_std::wasm_application_io::RenderOutputType::Svg(svg) => {
+					graphene_std::wasm_application_io::RenderOutputType::Svg { svg, image_data } => {
 						// Send to frontend
+						responses.add(FrontendMessage::UpdateImageData { image_data });
 						responses.add(FrontendMessage::UpdateDocumentArtwork { svg });
 					}
 					graphene_std::wasm_application_io::RenderOutputType::CanvasFrame(frame) => {

frontend/wasm/src/editor_api.rs

@@ -17,12 +17,16 @@ use editor::messages::prelude::*;
 use editor::messages::tool::tool_messages::tool_prelude::WidgetId;
 use graph_craft::document::NodeId;
 use graphene_std::raster::color::Color;
+use graphene_std::raster::{Image, TransformImage};
+use js_sys::{Object, Reflect};
 use serde::Serialize;
 use serde_wasm_bindgen::{self, from_value};
 use std::cell::RefCell;
 use std::sync::atomic::Ordering;
 use std::time::Duration;
+use wasm_bindgen::JsCast;
 use wasm_bindgen::prelude::*;
+use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData, window};
 
 /// Set the random seed used by the editor by calling this from JS upon initialization.
 /// This is necessary because WASM doesn't have a random number generator.
@@ -37,6 +41,73 @@ pub fn wasm_memory() -> JsValue {
 	wasm_bindgen::memory()
 }
 
+fn render_image_data_to_canvases(image_data: &[(u64, Image<Color>, TransformImage)]) {
+	let window = match window() {
+		Some(window) => window,
+		None => {
+			error!("Cannot render canvas: window object not found");
+			return;
+		}
+	};
+	let document = window.document().expect("window should have a document");
+	let window_obj = Object::from(window);
+	let image_canvases_key = JsValue::from_str("imageCanvases");
+
+	let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) {
+		Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj,
+		_ => {
+			let new_obj = Object::new();
+			if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() {
+				error!("Failed to create and set imageCanvases object on window");
+				return;
+			}
+			new_obj.into()
+		}
+	};
+	let canvases_obj = Object::from(canvases_obj);
+
+	for (placeholder_id, image, _) in image_data.iter() {
+		if image.width == 0 || image.height == 0 {
+			continue;
+		}
+
+		let canvas: HtmlCanvasElement = document
+			.create_element("canvas")
+			.expect("Failed to create canvas element")
+			.dyn_into::<HtmlCanvasElement>()
+			.expect("Failed to cast element to HtmlCanvasElement");
+
+		canvas.set_width(image.width);
+		canvas.set_height(image.height);
+		let context: CanvasRenderingContext2d = canvas
+			.get_context("2d")
+			.expect("Failed to get 2d context")
+			.expect("2d context was not found")
+			.dyn_into::<CanvasRenderingContext2d>()
+			.expect("Failed to cast context to CanvasRenderingContext2d");
+		let u8_data: Vec<u8> = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect();
+		let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]);
+		match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) {
+			Ok(image_data_obj) => {
+				if context.put_image_data(&image_data_obj, 0.0, 0.0).is_err() {
+					error!("Failed to put image data on canvas for id: {}", placeholder_id);
+				}
+			}
+			Err(e) => {
+				error!("Failed to create ImageData for id: {}: {:?}", placeholder_id, e);
+			}
+		}
+
+		let canvas_name = format!("canvas{}", placeholder_id);
+		let js_key = JsValue::from_str(&canvas_name);
+		let js_value = JsValue::from(canvas);
+
+		if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() {
+			error!("Failed to set canvas '{}' on imageCanvases object", canvas_name);
+		}
+	}
+}
+
 // ============================================================================
 
 /// This struct is, via wasm-bindgen, used by JS to interact with the editor backend. It does this by calling functions, which are `impl`ed
@@ -88,6 +159,11 @@ impl EditorHandle {
 
 	// Sends a FrontendMessage to JavaScript
 	fn send_frontend_message_to_js(&self, mut message: FrontendMessage) {
+		if let FrontendMessage::UpdateImageData { ref image_data } = message {
+			render_image_data_to_canvases(image_data.as_slice());
+			return;
+		}
+
 		if let FrontendMessage::UpdateDocumentLayerStructure { data_buffer } = message {
 			message = FrontendMessage::UpdateDocumentLayerStructureJs { data_buffer: data_buffer.into() };
 		}

node-graph/gcore/src/raster.rs

@@ -15,6 +15,7 @@ pub mod color {
 pub mod image;
 
 pub use self::image::Image;
+pub use self::image::TransformImage;
 
 pub trait Bitmap {
 	type Pixel: Pixel;

node-graph/gcore/src/raster/image.rs

@@ -50,6 +50,14 @@ pub struct Image<P: Pixel> {
 	// TODO: Currently it is always anchored at the top left corner at (0, 0). The bottom right corner of the new origin field would correspond to (1, 1).
 }
 
+#[derive(Debug, Clone, dyn_any::DynAny, Default, PartialEq, serde::Serialize, serde::Deserialize, specta::Type)]
+pub struct TransformImage(pub DAffine2);
+
+impl Hash for TransformImage {
+	fn hash<H: std::hash::Hasher>(&self, _: &mut H) {}
+}
+
+
 impl<P: Pixel + Debug> Debug for Image<P> {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		let length = self.data.len();

node-graph/graph-craft/src/document/value.rs

@@ -7,6 +7,7 @@ pub use glam::{DAffine2, DVec2, IVec2, UVec2};
 use graphene_application_io::SurfaceFrame;
 use graphene_brush::brush_cache::BrushCache;
 use graphene_brush::brush_stroke::BrushStroke;
+use graphene_core::raster::{Image, TransformImage};
 use graphene_core::raster_types::CPU;
 use graphene_core::transform::ReferencePoint;
 use graphene_core::uuid::NodeId;
@@ -425,10 +426,10 @@ pub struct RenderOutput {
 	pub metadata: RenderMetadata,
 }
 
-#[derive(Debug, Clone, PartialEq, dyn_any::DynAny, Hash, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Hash, PartialEq, dyn_any::DynAny, serde::Serialize, serde::Deserialize)]
 pub enum RenderOutputType {
 	CanvasFrame(SurfaceFrame),
-	Svg(String),
+	Svg { svg: String, image_data: Vec<(u64, Image<Color>, TransformImage)> },
 	Image(Vec<u8>),
 }
 

node-graph/graph-craft/src/wasm_application_io.rs

@@ -3,6 +3,7 @@ use graphene_application_io::{ApplicationError, ApplicationIo, ResourceFuture, S
 #[cfg(target_arch = "wasm32")]
 use js_sys::{Object, Reflect};
 use std::collections::HashMap;
+use std::hash::Hash;
 use std::sync::Arc;
 #[cfg(target_arch = "wasm32")]
 use std::sync::atomic::AtomicU64;

node-graph/gstd/src/wasm_application_io.rs

@@ -30,35 +30,6 @@ async fn create_surface<'a: 'n>(_: impl Ctx, editor: &'a WasmEditorApi) -> Arc<W
 	Arc::new(editor.application_io.as_ref().unwrap().create_window())
 }
 
-// TODO: Fix and reenable in order to get the 'Draw Canvas' node working again.
-// #[cfg(target_arch = "wasm32")]
-// use wasm_bindgen::Clamped;
-//
-// #[node_macro::node(category("Debug: GPU"))]
-// #[cfg(target_arch = "wasm32")]
-// async fn draw_image_frame(
-// 	_: impl Ctx,
-// 	image: RasterDataTable<graphene_core::raster::SRGBA8>,
-// 	surface_handle: Arc<WasmSurfaceHandle>,
-// ) -> graphene_core::application_io::SurfaceHandleFrame<HtmlCanvasElement> {
-// 	let image = image.instance_ref_iter().next().unwrap().instance;
-// 	let image_data = image.image.data;
-// 	let array: Clamped<&[u8]> = Clamped(bytemuck::cast_slice(image_data.as_slice()));
-// 	if image.image.width > 0 && image.image.height > 0 {
-// 		let canvas = &surface_handle.surface;
-// 		canvas.set_width(image.image.width);
-// 		canvas.set_height(image.image.height);
-// 		// TODO: replace "2d" with "bitmaprenderer" once we switch to ImageBitmap (lives on gpu) from RasterData (lives on cpu)
-// 		let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::<CanvasRenderingContext2d>().unwrap();
-// 		let image_data = web_sys::ImageData::new_with_u8_clamped_array_and_sh(array, image.image.width, image.image.height).expect("Failed to construct RasterData");
-// 		context.put_image_data(&image_data, 0., 0.).unwrap();
-// 	}
-// 	graphene_core::application_io::SurfaceHandleFrame {
-// 		surface_handle,
-// 		transform: image.transform,
-// 	}
-// }
-
 #[node_macro::node(category("Web Request"))]
 async fn load_resource<'a: 'n>(_: impl Ctx, _primary: (), #[scope("editor-api")] editor: &'a WasmEditorApi, #[name("URL")] url: String) -> Arc<[u8]> {
 	let Some(api) = editor.application_io.as_ref() else {
@@ -112,7 +83,9 @@ fn render_svg(data: impl GraphicElementRendered, mut render: SvgRender, render_p
 
 	render.wrap_with_transform(footprint.transform, Some(footprint.resolution.as_dvec2()));
 
-	RenderOutputType::Svg(render.svg.to_svg_string())
+	let svg = render.svg.to_svg_string();
+	let image_data = render.image_data;
+	RenderOutputType::Svg { svg, image_data }
 }
 
 #[cfg(feature = "vello")]