diff --git a/Cargo.lock b/Cargo.lock index 5487e27d28..46f97214fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2439,6 +2439,7 @@ version = "0.0.0" dependencies = [ "bezier-rs", "bitflags 2.9.1", + "bytemuck", "derivative", "dyn-any", "env_logger", diff --git a/editor/Cargo.toml b/editor/Cargo.toml index 1f6af599ca..5b8dbd425b 100644 --- a/editor/Cargo.toml +++ b/editor/Cargo.toml @@ -46,6 +46,7 @@ num_enum = { workspace = true } usvg = { workspace = true } once_cell = { workspace = true } web-sys = { workspace = true } +bytemuck = { workspace = true } # Required dependencies spin = "0.9.8" diff --git a/editor/src/messages/frontend/frontend_message.rs b/editor/src/messages/frontend/frontend_message.rs index c24ebc405c..cc722ffad7 100644 --- a/editor/src/messages/frontend/frontend_message.rs +++ b/editor/src/messages/frontend/frontend_message.rs @@ -8,6 +8,8 @@ use crate::messages::portfolio::document::utility_types::wires::{WirePath, WireP use crate::messages::prelude::*; use crate::messages::tool::utility_types::HintData; use graph_craft::document::NodeId; +use graphene_std::raster::Image; +use graphene_std::raster::TransformImage; use graphene_std::raster::color::Color; use graphene_std::text::Font; @@ -179,6 +181,9 @@ pub enum FrontendMessage { UpdateDocumentArtwork { svg: String, }, + UpdateImageData { + image_data: Vec<(u64, Image, TransformImage)>, + }, UpdateDocumentBarLayout { #[serde(rename = "layoutTarget")] layout_target: LayoutTarget, diff --git a/editor/src/messages/prelude.rs b/editor/src/messages/prelude.rs index 72a6cb9ba7..5a57f48bd9 100644 --- a/editor/src/messages/prelude.rs +++ b/editor/src/messages/prelude.rs @@ -15,13 +15,7 @@ pub use crate::messages::input_mapper::key_mapping::{KeyMappingMessage, KeyMappi pub use crate::messages::input_mapper::{InputMapperMessage, InputMapperMessageContext, InputMapperMessageDiscriminant, InputMapperMessageHandler}; pub use crate::messages::input_preprocessor::{InputPreprocessorMessage, InputPreprocessorMessageContext, InputPreprocessorMessageDiscriminant, InputPreprocessorMessageHandler}; pub use crate::messages::layout::{LayoutMessage, LayoutMessageDiscriminant, LayoutMessageHandler}; -pub use crate::messages::portfolio::document::graph_operation::{GraphOperationMessage, GraphOperationMessageContext, GraphOperationMessageDiscriminant, GraphOperationMessageHandler}; -pub use crate::messages::portfolio::document::navigation::{NavigationMessage, NavigationMessageContext, NavigationMessageDiscriminant, NavigationMessageHandler}; -pub use crate::messages::portfolio::document::node_graph::{NodeGraphMessage, NodeGraphMessageDiscriminant, NodeGraphMessageHandler}; -pub use crate::messages::portfolio::document::overlays::{OverlaysMessage, OverlaysMessageContext, OverlaysMessageDiscriminant, OverlaysMessageHandler}; -pub use crate::messages::portfolio::document::properties_panel::{PropertiesPanelMessage, PropertiesPanelMessageDiscriminant, PropertiesPanelMessageHandler}; -pub use crate::messages::portfolio::document::{DocumentMessage, DocumentMessageContext, DocumentMessageDiscriminant, DocumentMessageHandler}; -pub use crate::messages::portfolio::menu_bar::{MenuBarMessage, MenuBarMessageDiscriminant, MenuBarMessageHandler}; +pub use crate::messages::portfolio::document::graph_operation::{GraphOperationMessage, GraphOperationMessageContext, GraphOperationMessageDiscriminant, GraphOperationMessageHandler}; pub use crate::messages::portfolio::document::navigation::{NavigationMessage, NavigationMessageContext, NavigationMessageDiscriminant, NavigationMessageHandler}; pub use crate::messages::portfolio::document::node_graph::{NodeGraphMessage, NodeGraphMessageDiscriminant, NodeGraphMessageHandler}; pub use crate::messages::portfolio::document::overlays::{OverlaysMessage, OverlaysMessageContext, OverlaysMessageDiscriminant, OverlaysMessageHandler}; pub use crate::messages::portfolio::document::properties_panel::{PropertiesPanelMessage, PropertiesPanelMessageDiscriminant, PropertiesPanelMessageHandler}; pub use crate::messages::portfolio::document::{DocumentMessage, DocumentMessageContext, DocumentMessageDiscriminant, DocumentMessageHandler}; pub use crate::messages::portfolio::menu_bar::{MenuBarMessage, MenuBarMessageDiscriminant, MenuBarMessageHandler}; pub use crate::messages::portfolio::spreadsheet::{SpreadsheetMessage, SpreadsheetMessageDiscriminant}; pub use crate::messages::portfolio::{PortfolioMessage, PortfolioMessageContext, PortfolioMessageDiscriminant, PortfolioMessageHandler}; pub use crate::messages::preferences::{PreferencesMessage, PreferencesMessageDiscriminant, PreferencesMessageHandler}; diff --git a/editor/src/node_graph_executor.rs b/editor/src/node_graph_executor.rs index 8c6a7d13a3..a3067b436b 100644 --- a/editor/src/node_graph_executor.rs +++ b/editor/src/node_graph_executor.rs @@ -213,7 +213,7 @@ impl NodeGraphExecutor { fn export(&self, node_graph_output: TaggedValue, export_config: ExportConfig, responses: &mut VecDeque) -> Result<(), String> { let TaggedValue::RenderOutput(RenderOutput { - data: graphene_std::wasm_application_io::RenderOutputType::Svg(svg), + data: graphene_std::wasm_application_io::RenderOutputType::Svg { svg, .. }, .. }) = node_graph_output else { @@ -350,8 +350,9 @@ impl NodeGraphExecutor { match node_graph_output { TaggedValue::RenderOutput(render_output) => { match render_output.data { - graphene_std::wasm_application_io::RenderOutputType::Svg(svg) => { + graphene_std::wasm_application_io::RenderOutputType::Svg { svg, image_data } => { // Send to frontend + responses.add(FrontendMessage::UpdateImageData { image_data }); responses.add(FrontendMessage::UpdateDocumentArtwork { svg }); } graphene_std::wasm_application_io::RenderOutputType::CanvasFrame(frame) => { diff --git a/frontend/wasm/src/editor_api.rs b/frontend/wasm/src/editor_api.rs index c8056e4efc..7c7676e355 100644 --- a/frontend/wasm/src/editor_api.rs +++ b/frontend/wasm/src/editor_api.rs @@ -17,12 +17,16 @@ use editor::messages::prelude::*; use editor::messages::tool::tool_messages::tool_prelude::WidgetId; use graph_craft::document::NodeId; use graphene_std::raster::color::Color; +use graphene_std::raster::{Image, TransformImage}; +use js_sys::{Object, Reflect}; use serde::Serialize; use serde_wasm_bindgen::{self, from_value}; use std::cell::RefCell; use std::sync::atomic::Ordering; use std::time::Duration; +use wasm_bindgen::JsCast; use wasm_bindgen::prelude::*; +use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData, window}; /// Set the random seed used by the editor by calling this from JS upon initialization. /// This is necessary because WASM doesn't have a random number generator. @@ -37,6 +41,73 @@ pub fn wasm_memory() -> JsValue { wasm_bindgen::memory() } +fn render_image_data_to_canvases(image_data: &[(u64, Image, TransformImage)]) { + let window = match window() { + Some(window) => window, + None => { + error!("Cannot render canvas: window object not found"); + return; + } + }; + let document = window.document().expect("window should have a document"); + let window_obj = Object::from(window); + let image_canvases_key = JsValue::from_str("imageCanvases"); + + let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) { + Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj, + _ => { + let new_obj = Object::new(); + if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() { + error!("Failed to create and set imageCanvases object on window"); + return; + } + new_obj.into() + } + }; + let canvases_obj = Object::from(canvases_obj); + + for (placeholder_id, image, _) in image_data.iter() { + if image.width == 0 || image.height == 0 { + continue; + } + + let canvas: HtmlCanvasElement = document + .create_element("canvas") + .expect("Failed to create canvas element") + .dyn_into::() + .expect("Failed to cast element to HtmlCanvasElement"); + + canvas.set_width(image.width); + canvas.set_height(image.height); + let context: CanvasRenderingContext2d = canvas + .get_context("2d") + .expect("Failed to get 2d context") + .expect("2d context was not found") + .dyn_into::() + .expect("Failed to cast context to CanvasRenderingContext2d"); + let u8_data: Vec = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect(); + let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]); + match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) { + Ok(image_data_obj) => { + if context.put_image_data(&image_data_obj, 0.0, 0.0).is_err() { + error!("Failed to put image data on canvas for id: {}", placeholder_id); + } + } + Err(e) => { + error!("Failed to create ImageData for id: {}: {:?}", placeholder_id, e); + } + } + + let canvas_name = format!("canvas{}", placeholder_id); + let js_key = JsValue::from_str(&canvas_name); + let js_value = JsValue::from(canvas); + + if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() { + error!("Failed to set canvas '{}' on imageCanvases object", canvas_name); + } + } +} + // ============================================================================ /// This struct is, via wasm-bindgen, used by JS to interact with the editor backend. It does this by calling functions, which are `impl`ed @@ -88,6 +159,11 @@ impl EditorHandle { // Sends a FrontendMessage to JavaScript fn send_frontend_message_to_js(&self, mut message: FrontendMessage) { + if let FrontendMessage::UpdateImageData { ref image_data } = message { + render_image_data_to_canvases(image_data.as_slice()); + return; + } + if let FrontendMessage::UpdateDocumentLayerStructure { data_buffer } = message { message = FrontendMessage::UpdateDocumentLayerStructureJs { data_buffer: data_buffer.into() }; } diff --git a/node-graph/gcore/src/raster.rs b/node-graph/gcore/src/raster.rs index 60106bdc70..d38fbf467b 100644 --- a/node-graph/gcore/src/raster.rs +++ b/node-graph/gcore/src/raster.rs @@ -15,6 +15,7 @@ pub mod color { pub mod image; pub use self::image::Image; +pub use self::image::TransformImage; pub trait Bitmap { type Pixel: Pixel; diff --git a/node-graph/gcore/src/raster/image.rs b/node-graph/gcore/src/raster/image.rs index e93fe60ba2..92b49ee462 100644 --- a/node-graph/gcore/src/raster/image.rs +++ b/node-graph/gcore/src/raster/image.rs @@ -50,6 +50,14 @@ pub struct Image { // TODO: Currently it is always anchored at the top left corner at (0, 0). The bottom right corner of the new origin field would correspond to (1, 1). } +#[derive(Debug, Clone, dyn_any::DynAny, Default, PartialEq, serde::Serialize, serde::Deserialize, specta::Type)] +pub struct TransformImage(pub DAffine2); + +impl Hash for TransformImage { + fn hash(&self, _: &mut H) {} +} + + impl Debug for Image

{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let length = self.data.len(); diff --git a/node-graph/graph-craft/src/document/value.rs b/node-graph/graph-craft/src/document/value.rs index c8c896290c..4d04d9553b 100644 --- a/node-graph/graph-craft/src/document/value.rs +++ b/node-graph/graph-craft/src/document/value.rs @@ -7,6 +7,7 @@ pub use glam::{DAffine2, DVec2, IVec2, UVec2}; use graphene_application_io::SurfaceFrame; use graphene_brush::brush_cache::BrushCache; use graphene_brush::brush_stroke::BrushStroke; +use graphene_core::raster::{Image, TransformImage}; use graphene_core::raster_types::CPU; use graphene_core::transform::ReferencePoint; use graphene_core::uuid::NodeId; @@ -425,10 +426,10 @@ pub struct RenderOutput { pub metadata: RenderMetadata, } -#[derive(Debug, Clone, PartialEq, dyn_any::DynAny, Hash, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Hash, PartialEq, dyn_any::DynAny, serde::Serialize, serde::Deserialize)] pub enum RenderOutputType { CanvasFrame(SurfaceFrame), - Svg(String), + Svg { svg: String, image_data: Vec<(u64, Image, TransformImage)> }, Image(Vec), } diff --git a/node-graph/graph-craft/src/wasm_application_io.rs b/node-graph/graph-craft/src/wasm_application_io.rs index 1d11744aa4..f4398aef54 100644 --- a/node-graph/graph-craft/src/wasm_application_io.rs +++ b/node-graph/graph-craft/src/wasm_application_io.rs @@ -3,6 +3,7 @@ use graphene_application_io::{ApplicationError, ApplicationIo, ResourceFuture, S #[cfg(target_arch = "wasm32")] use js_sys::{Object, Reflect}; use std::collections::HashMap; +use std::hash::Hash; use std::sync::Arc; #[cfg(target_arch = "wasm32")] use std::sync::atomic::AtomicU64; diff --git a/node-graph/gstd/src/wasm_application_io.rs b/node-graph/gstd/src/wasm_application_io.rs index ae03edd425..daa083041b 100644 --- a/node-graph/gstd/src/wasm_application_io.rs +++ b/node-graph/gstd/src/wasm_application_io.rs @@ -30,35 +30,6 @@ async fn create_surface<'a: 'n>(_: impl Ctx, editor: &'a WasmEditorApi) -> Arc, -// surface_handle: Arc, -// ) -> graphene_core::application_io::SurfaceHandleFrame { -// let image = image.instance_ref_iter().next().unwrap().instance; -// let image_data = image.image.data; -// let array: Clamped<&[u8]> = Clamped(bytemuck::cast_slice(image_data.as_slice())); -// if image.image.width > 0 && image.image.height > 0 { -// let canvas = &surface_handle.surface; -// canvas.set_width(image.image.width); -// canvas.set_height(image.image.height); -// // TODO: replace "2d" with "bitmaprenderer" once we switch to ImageBitmap (lives on gpu) from RasterData (lives on cpu) -// let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::().unwrap(); -// let image_data = web_sys::ImageData::new_with_u8_clamped_array_and_sh(array, image.image.width, image.image.height).expect("Failed to construct RasterData"); -// context.put_image_data(&image_data, 0., 0.).unwrap(); -// } -// graphene_core::application_io::SurfaceHandleFrame { -// surface_handle, -// transform: image.transform, -// } -// } - #[node_macro::node(category("Web Request"))] async fn load_resource<'a: 'n>(_: impl Ctx, _primary: (), #[scope("editor-api")] editor: &'a WasmEditorApi, #[name("URL")] url: String) -> Arc<[u8]> { let Some(api) = editor.application_io.as_ref() else { @@ -112,7 +83,9 @@ fn render_svg(data: impl GraphicElementRendered, mut render: SvgRender, render_p render.wrap_with_transform(footprint.transform, Some(footprint.resolution.as_dvec2())); - RenderOutputType::Svg(render.svg.to_svg_string()) + let svg = render.svg.to_svg_string(); + let image_data = render.image_data; + RenderOutputType::Svg { svg, image_data } } #[cfg(feature = "vello")] diff --git a/node-graph/gsvg-renderer/src/renderer.rs b/node-graph/gsvg-renderer/src/renderer.rs index 216a9b666f..efa8570720 100644 --- a/node-graph/gsvg-renderer/src/renderer.rs +++ b/node-graph/gsvg-renderer/src/renderer.rs @@ -8,7 +8,7 @@ use graphene_core::bounds::BoundingBox; use graphene_core::color::Color; use graphene_core::instances::Instance; use graphene_core::math::quad::Quad; -use graphene_core::raster::Image; +use graphene_core::raster::{Image, TransformImage}; use graphene_core::raster_types::{CPU, GPU, RasterDataTable}; use graphene_core::render_complexity::RenderComplexity; use graphene_core::transform::{Footprint, Transform}; @@ -51,7 +51,7 @@ pub struct SvgRender { pub svg: Vec, pub svg_defs: String, pub transform: DAffine2, - pub image_data: Vec<(u64, Image)>, + pub image_data: Vec<(u64, Image, TransformImage)>, indent: usize, } @@ -173,6 +173,10 @@ impl RenderParams { let alignment_parent_transform = Some(transform); Self { alignment_parent_transform, ..*self } } + + pub fn to_canvas(&self) -> bool { + !self.for_export && !self.thumbnail + } } pub fn format_transform_matrix(transform: DAffine2) -> String { @@ -937,40 +941,78 @@ impl GraphicElementRendered for RasterDataTable { fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) { for instance in self.instance_ref_iter() { let transform = *instance.transform; - let image = &instance.instance; + if image.data.is_empty() { - return; + continue; } - let base64_string = image.base64_string.clone().unwrap_or_else(|| { - use base64::Engine; + if render_params.to_canvas() { + let id = generate_uuid(); + let mut transform_values = transform.to_scale_angle_translation(); + render.image_data.push((id, image.data().clone(), TransformImage(transform))); + render.parent_tag( + "foreignObject", + |attributes| { + let size = DVec2::new(image.width as f64, image.height as f64); + transform_values.0 /= size; + let matrix = DAffine2::from_scale_angle_translation(transform_values.0, transform_values.1, transform_values.2); + let matrix = format_transform_matrix(matrix); + if !matrix.is_empty() { + attributes.push("transform", matrix); + } + attributes.push("width", size.x.to_string()); + attributes.push("height", size.y.to_string()); - let output = image.to_png(); - let preamble = "data:image/png;base64,"; - let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4); - base64_string.push_str(preamble); - base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string); - base64_string - }); - render.leaf_tag("image", |attributes| { - attributes.push("width", 1.to_string()); - attributes.push("height", 1.to_string()); - attributes.push("preserveAspectRatio", "none"); - attributes.push("href", base64_string); - let matrix = format_transform_matrix(transform); - if !matrix.is_empty() { - attributes.push("transform", matrix); - } - let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill }; - let opacity = instance.alpha_blending.opacity * factor; - if opacity < 1. { - attributes.push("opacity", opacity.to_string()); - } - if instance.alpha_blending.blend_mode != BlendMode::default() { - attributes.push("style", instance.alpha_blending.blend_mode.render()); - } - }); + let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill }; + let opacity = instance.alpha_blending.opacity * factor; + if opacity < 1. { + attributes.push("opacity", opacity.to_string()); + } + if instance.alpha_blending.blend_mode != BlendMode::default() { + attributes.push("style", instance.alpha_blending.blend_mode.render()); + } + }, + |render| { + render.leaf_tag( + "img", // Must be a self-closing tag + |attributes| { + attributes.push("data-canvas-placeholder", format!("canvas{}", id)); + }, + ) + }, + ); + } else { + let base64_string = image.base64_string.clone().unwrap_or_else(|| { + use base64::Engine; + + let output = image.to_png(); + let preamble = "data:image/png;base64,"; + let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4); + base64_string.push_str(preamble); + base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string); + base64_string + }); + + render.leaf_tag("image", |attributes| { + attributes.push("width", "1"); + attributes.push("height", "1"); + attributes.push("preserveAspectRatio", "none"); + attributes.push("href", base64_string); + let matrix = format_transform_matrix(transform); + if !matrix.is_empty() { + attributes.push("transform", matrix); + } + let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill }; + let opacity = instance.alpha_blending.opacity * factor; + if opacity < 1. { + attributes.push("opacity", opacity.to_string()); + } + if instance.alpha_blending.blend_mode != BlendMode::default() { + attributes.push("style", instance.alpha_blending.blend_mode.render()); + } + }); + } } }