Inspector V2: Capture Tools updates (#17713)

<img width="465" height="642" alt="image"
src="https://github.com/user-attachments/assets/3dc0d199-49ef-4ed0-a059-12e6c659ea19"
/>

### Changes
- Add GIF recorder
- Add scene replay 
- Some aesthetics... let me know what you think.
- Separate capture screenshot, capture equirectangular, and record video
into their own sections.
  - Always show width/height of screenshot
- Marry capture screenshot with capture RTT (for a UI, they seem
functionally equivalent, right?)

### Notes
*State in components vs. service*: I've kept the functionality in the
components for now, instead of in a service, both because it was simpler
and because these components don't really need to persist state outside
of the Inspector's lifetime. (Unlike the glTF loader options and glTF
validator in the Import Tools...)

*GIF module*: There aren't many options out there for recording GIFs on
the web. `gif.js` seems to be standard, and it's what we had before. I
installed it as a devDependency in the Inspector v2, assuming that it
will be built into the bundled package. Hope this is OK, but let me
know.

Author: alexchuber

package-lock.json

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import type { Scene } from "../scene";
+import type { IDisposable, Scene } from "../scene";
 import type { Nullable } from "../types";
 import { SceneSerializer } from "./sceneSerializer";
 import { Mesh } from "../Meshes/mesh";
@@ -19,7 +19,7 @@ import { SerializationHelper } from "./decorators.serialization";
 /**
  * Class used to record delta files between 2 scene states
  */
-export class SceneRecorder {
+export class SceneRecorder implements IDisposable {
     private _trackedScene: Nullable<Scene> = null;
     private _savedJSON: any;
 
@@ -194,6 +194,14 @@ export class SceneRecorder {
         }
     }
 
+    /**
+     * Dispose the recorder.
+     */
+    public dispose() {
+        this._trackedScene = null;
+        this._savedJSON = null;
+    }
+
     private static GetShadowGeneratorById(scene: Scene, id: string) {
         const allGenerators = scene.lights.map((l) => l.getShadowGenerators());
 

packages/dev/core/src/Misc/sceneRecorder.ts

@@ -29,6 +29,7 @@
         "@tools/gui-editor": "1.0.0",
         "@types/react": "^18.0.0",
         "@types/react-dom": "^18.0.0",
+        "gif.js.optimized": "^1.0.1",
         "html-webpack-plugin": "^5.5.0",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",

packages/dev/inspector-v2/package.json

@@ -0,0 +1,23 @@
+import { ButtonLine } from "shared-ui-components/fluent/hoc/buttonLine";
+import { useCallback } from "react";
+import type { FunctionComponent } from "react";
+import type { Scene } from "core/scene";
+import { captureEquirectangularFromScene } from "core/Misc/equirectangularCapture";
+import { CameraRegular } from "@fluentui/react-icons";
+import { FrameGraphUtils } from "core/FrameGraph/frameGraphUtils";
+
+export const EquirectangularCaptureTool: FunctionComponent<{ scene: Scene }> = ({ scene }) => {
+    const captureEquirectangularAsync = useCallback(async () => {
+        const currentActiveCamera = scene.activeCamera;
+        if (!currentActiveCamera && scene.frameGraph) {
+            scene.activeCamera = FrameGraphUtils.FindMainCamera(scene.frameGraph);
+        }
+        if (scene.activeCamera) {
+            await captureEquirectangularFromScene(scene, { size: 1024, filename: "equirectangular_capture.png" });
+        }
+        // eslint-disable-next-line require-atomic-updates
+        scene.activeCamera = currentActiveCamera;
+    }, [scene]);
+
+    return <ButtonLine label="Capture Equirectangular" icon={CameraRegular} onClick={captureEquirectangularAsync} />;
+};

packages/dev/inspector-v2/src/components/tools/capture/equirectangularCaptureTool.tsx

@@ -0,0 +1,26 @@
+// See https://github.com/jnordberg/gif.js?tab=readme-ov-file#options
+declare module "gif.js.optimized" {
+    class GIF {
+        constructor(options?: {
+            // pixel sample interval, lower is better
+            quality?: number;
+            // number of web workers to spawn
+            workers?: number;
+            // url to load worker script from
+            workerScript?: string;
+        });
+        addFrame(
+            image: HTMLImageElement | HTMLCanvasElement | CanvasRenderingContext2D,
+            options?: {
+                // frame delay (ms)
+                delay?: number;
+                // copy the pixel data
+                copy?: boolean;
+            }
+        ): void;
+        on(event: "finished", callback: (blob: Blob) => void): void;
+        render(): void;
+    }
+
+    export default GIF;
+}

packages/dev/inspector-v2/src/components/tools/capture/gif.js.optimized.d.ts

@@ -0,0 +1,152 @@
+import { ButtonLine } from "shared-ui-components/fluent/hoc/buttonLine";
+import { useState, useCallback, useEffect } from "react";
+import type { Scene } from "core/scene";
+import { SyncedSliderPropertyLine } from "shared-ui-components/fluent/hoc/propertyLines/syncedSliderPropertyLine";
+import { Collapse } from "shared-ui-components/fluent/primitives/collapse";
+import { RecordRegular, RecordStopRegular } from "@fluentui/react-icons";
+import { Tools } from "core/Misc/tools";
+import { Label } from "@fluentui/react-components";
+import { MakeLazyComponent } from "shared-ui-components/fluent/primitives/lazyComponent";
+import type gif from "gif.js.optimized";
+import type { Observer } from "core/Misc/observable";
+
+type RecordingSession =
+    | {
+          state: "Recording";
+          gif: gif;
+          captureObserver: Observer<Scene>;
+          previousHardwareScaling: number;
+      }
+    | {
+          state: "Rendering";
+          gif: gif;
+      }
+    | {
+          state: "Idle";
+      };
+
+export const GIFCaptureTool = MakeLazyComponent(
+    async () => {
+        const gif = (await import("gif.js.optimized")).default;
+
+        // TODO: Figure out how to grab this from NPM package instead of CDN
+        const workerContent = await Tools.LoadFileAsync("https://cdn.jsdelivr.net/gh//terikon/gif.js.optimized@0.1.6/dist/gif.worker.js");
+        const workerBlob = new Blob([workerContent], { type: "application/javascript" });
+        const workerUrl = URL.createObjectURL(workerBlob);
+
+        return ({ scene }: { scene: Scene }) => {
+            const [recordingSession, setRecordingSession] = useState<RecordingSession>({ state: "Idle" });
+            const [targetWidth, setTargetWidth] = useState(512);
+            const [frequency, setFrequency] = useState(200);
+
+            useEffect(() => {
+                return () => {
+                    if (recordingSession.state === "Recording") {
+                        // Reset session resources if component is unmounted
+                        scene.onAfterRenderObservable.remove(recordingSession.captureObserver);
+                        scene.getEngine().setHardwareScalingLevel(recordingSession.previousHardwareScaling);
+                    }
+                };
+            }, [recordingSession, scene]);
+
+            // Use functional setState to guard against multiple rapid clicks
+            const startRecording = useCallback(() => {
+                setRecordingSession((currentSession) => {
+                    // If already recording/rendering, don't start a new session
+                    if (currentSession.state !== "Idle") {
+                        return currentSession;
+                    }
+
+                    const engine = scene.getEngine();
+                    const canvas = engine.getRenderingCanvas();
+                    if (!canvas) {
+                        return currentSession;
+                    }
+
+                    const gifInstance = new gif({
+                        workers: 2,
+                        quality: 10,
+                        workerScript: workerUrl,
+                    });
+
+                    // Adjust hardware scaling to match desired width
+                    const previousHardwareScaling = engine.getHardwareScalingLevel();
+                    engine.setHardwareScalingLevel(engine.getRenderWidth() / (targetWidth * globalThis.devicePixelRatio) || 1);
+
+                    // Capture frames after each render
+                    let lastCaptureTime = 0;
+                    const captureObserver = scene.onAfterRenderObservable.add(() => {
+                        const now = Date.now();
+                        if (now - lastCaptureTime >= frequency && gifInstance) {
+                            lastCaptureTime = now;
+                            gifInstance.addFrame(canvas, { delay: 1, copy: true });
+                        }
+                    });
+
+                    return {
+                        state: "Recording",
+                        gif: gifInstance,
+                        captureObserver: captureObserver,
+                        previousHardwareScaling: previousHardwareScaling,
+                    };
+                });
+            }, [scene, targetWidth, frequency]);
+
+            const stopRecording = useCallback(() => {
+                setRecordingSession((currentSession) => {
+                    if (currentSession.state !== "Recording") {
+                        return currentSession;
+                    }
+
+                    // Remove the frame capture observer
+                    scene.onAfterRenderObservable.remove(currentSession.captureObserver);
+
+                    // Restore previous hardware scaling
+                    scene.getEngine().setHardwareScalingLevel(currentSession.previousHardwareScaling);
+
+                    currentSession.gif.on("finished", (blob: Blob) => {
+                        // Download the rendered GIF
+                        Tools.Download(blob, "recording.gif");
+
+                        // Reset state
+                        setRecordingSession({ state: "Idle" });
+                    });
+
+                    // Start rendering the GIF
+                    currentSession.gif.render();
+
+                    return { state: "Rendering", gif: currentSession.gif };
+                });
+            }, [scene]);
+
+            return (
+                <>
+                    {recordingSession.state === "Idle" && <ButtonLine label="Record GIF" icon={RecordRegular} onClick={startRecording} />}
+                    {recordingSession.state === "Recording" && <ButtonLine label="Stop" icon={RecordStopRegular} onClick={stopRecording} />}
+                    {recordingSession.state === "Rendering" && <Label>Creating the GIF file...</Label>}
+                    <Collapse visible={recordingSession.state === "Idle"}>
+                        <SyncedSliderPropertyLine
+                            label="Resolution"
+                            description="The pixel width of the output. The height will be adjusted accordingly to maintain the aspect ratio."
+                            value={targetWidth}
+                            onChange={(value) => setTargetWidth(Math.floor(value))}
+                            min={128}
+                            max={2048}
+                            step={128}
+                        />
+                        <SyncedSliderPropertyLine
+                            label="Frequency (ms)"
+                            description="The time interval in milliseconds between each capture of the scene."
+                            value={frequency}
+                            onChange={(value) => setFrequency(Math.floor(value))}
+                            min={50}
+                            max={1000}
+                            step={50}
+                        />
+                    </Collapse>
+                </>
+            );
+        };
+    },
+    { spinnerSize: "extra-tiny", spinnerLabel: "Loading..." }
+);

packages/dev/inspector-v2/src/components/tools/capture/gifCaptureTool.tsx

@@ -0,0 +1,65 @@
+import { ButtonLine } from "shared-ui-components/fluent/hoc/buttonLine";
+import { useState, useCallback } from "react";
+import type { FunctionComponent } from "react";
+import type { Scene } from "core/scene";
+import { RecordRegular, SaveRegular, ArrowDownloadRegular } from "@fluentui/react-icons";
+import { SceneRecorder } from "core/Misc/sceneRecorder";
+import { Tools } from "core/Misc/tools";
+import { FileUploadLine } from "shared-ui-components/fluent/hoc/fileUploadLine";
+import { Label } from "@fluentui/react-components";
+import { Logger } from "core/Misc/logger";
+import { useResource } from "../../../hooks/resourceHooks";
+
+export const SceneReplayTool: FunctionComponent<{ scene: Scene }> = ({ scene }) => {
+    const [isRecording, setIsRecording] = useState(false);
+    const sceneRecorder = useResource(() => new SceneRecorder());
+
+    const startRecording = useCallback(() => {
+        sceneRecorder.track(scene);
+        setIsRecording(true);
+    }, [scene]);
+
+    const exportReplay = useCallback(() => {
+        const content = JSON.stringify(sceneRecorder.getDelta());
+        const blob = new Blob([content], { type: "application/json" });
+        Tools.Download(blob, "replay_delta.json");
+        setIsRecording(false);
+    }, []);
+
+    const applyDelta = useCallback(
+        (files: FileList) => {
+            const file = files[0];
+            if (!file) {
+                return;
+            }
+
+            Tools.ReadFile(
+                file,
+                (data) => {
+                    try {
+                        const json = JSON.parse(data);
+                        SceneRecorder.ApplyDelta(json, scene);
+                    } catch (error) {
+                        Logger.Error("Failed to apply replay delta:" + error);
+                    }
+                },
+                undefined,
+                false
+            );
+        },
+        [scene]
+    );
+
+    return (
+        <>
+            {!isRecording && <ButtonLine label="Start Recording" icon={RecordRegular} onClick={startRecording} />}
+            {isRecording && (
+                <>
+                    <Label>Recording in progress...</Label>
+                    <ButtonLine label="Generate Delta File" icon={SaveRegular} onClick={exportReplay} />
+                </>
+            )}
+            <FileUploadLine label="Apply Delta File" icon={ArrowDownloadRegular} onClick={applyDelta} accept=".json" />
+        </>
+    );
+};

packages/dev/inspector-v2/src/components/tools/capture/sceneReplayTool.tsx

@@ -0,0 +1,47 @@
+import { ButtonLine } from "shared-ui-components/fluent/hoc/buttonLine";
+import { useState, useCallback } from "react";
+import type { FunctionComponent } from "react";
+import type { Scene } from "core/scene";
+import { SyncedSliderPropertyLine } from "shared-ui-components/fluent/hoc/propertyLines/syncedSliderPropertyLine";
+import { CameraRegular } from "@fluentui/react-icons";
+import { FrameGraphUtils } from "core/FrameGraph/frameGraphUtils";
+import { CreateScreenshotUsingRenderTargetAsync } from "core/Misc/screenshotTools";
+import { SwitchPropertyLine } from "shared-ui-components/fluent/hoc/propertyLines/switchPropertyLine";
+import { Collapse } from "shared-ui-components/fluent/primitives/collapse";
+
+export const ScreenshotTool: FunctionComponent<{ scene: Scene }> = ({ scene }) => {
+    const [precision, setPrecision] = useState<number>(1);
+    const [useCustomSize, setUseCustomSize] = useState<boolean>(false);
+    const [width, setWidth] = useState<number>(512);
+    const [height, setHeight] = useState<number>(512);
+
+    const captureScreenshot = useCallback(async () => {
+        const engine = scene.getEngine();
+        const camera = scene.frameGraph ? FrameGraphUtils.FindMainCamera(scene.frameGraph) : scene.activeCamera;
+        const screenshotSize = useCustomSize ? { width, height, precision } : { precision };
+
+        if (camera) {
+            await CreateScreenshotUsingRenderTargetAsync(engine, camera, screenshotSize, "image/png", undefined, undefined, "screenshot.png");
+        }
+    }, [useCustomSize, precision, width, height, scene]);
+
+    return (
+        <>
+            <ButtonLine label="Capture Screenshot" icon={CameraRegular} onClick={captureScreenshot} />
+            <SyncedSliderPropertyLine
+                label="Precision"
+                description="A multiplier allowing capture at a higher or lower resolution."
+                value={precision}
+                onChange={setPrecision}
+                min={0.1}
+                max={10}
+                step={0.1}
+            />
+            <SwitchPropertyLine label="Use Custom Size" value={useCustomSize} onChange={setUseCustomSize} />
+            <Collapse visible={useCustomSize}>
+                <SyncedSliderPropertyLine label="Width" description="The width of the screenshot in pixels. " value={width} onChange={setWidth} min={1} step={1} />
+                <SyncedSliderPropertyLine label="Height" description="The height of the screenshot in pixels." value={height} onChange={setHeight} min={1} step={1} />
+            </Collapse>
+        </>
+    );
+};