feat: Camera Projection Transforms

examples/camera_projection.py

@@ -0,0 +1,48 @@
+from math import atan, pi
+
+import numpy as np
+
+import scenex as snx
+from scenex.model._transform import Transform
+from scenex.utils import projections
+
+try:
+    from imageio.v2 import volread
+
+    url = "https://gitlab.com/scikit-image/data/-/raw/2cdc5ce89b334d28f06a58c9f0ca21aa6992a5ba/cells3d.tif"
+    data = np.asarray(volread(url)).astype(np.uint16)[:, 0, :, :]
+except ImportError:
+    data = np.random.randint(0, 2, (3, 3, 3)).astype(np.uint16)
+
+view = snx.View(
+    blending="default",
+    scene=snx.Scene(
+        children=[
+            snx.Volume(
+                data=data,
+                clims=(data.min(), data.max()),
+            ),
+        ]
+    ),
+)
+
+canvas = snx.show(view)
+
+# Translate the camera to the center of the volume, and distance the camera from the
+# volume in the z dimension (important for perspective transforms)
+view.camera.transform = Transform().translated((127.5, 127.5, 300))
+
+# view.camera.projection = projections.orthographic(
+#     1.1 * data.shape[1],
+#     1.1 * data.shape[2],
+#     1000,
+# )
+
+view.camera.projection = projections.perspective(
+    # TODO: Create a helper function for this.
+    fov=2 * atan(data.shape[1] / 2 / 300) * 180 / pi,
+    near=300,
+    far=1_000_000,  # Just need something big
+)
+
+snx.run()

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
     "Typing :: Typed",
 ]
-dependencies = ["cmap>=0.5", "numpy>=1.24", "psygnal>=0.11.1", "pydantic>=2.10"]
+dependencies = ["cmap>=0.5", "numpy>=1.24", "psygnal>=0.11.1", "pydantic>=2.10", "pylinalg"]
 
 [project.optional-dependencies]
 pygfx = ["pygfx>=0.9.0"]
@@ -129,7 +129,7 @@ module = ["rendercanvas.*"]
 follow_untyped_imports = true
 
 [[tool.mypy.overrides]]
-module = ["pygfx.*", "vispy.*", "wgpu.*"]
+module = ["pygfx.*", "vispy.*", "wgpu.*", "pylinalg.*"]
 ignore_missing_imports = true
 
 [tool.pydantic-mypy]

src/scenex/adaptors/_base.py

@@ -120,11 +120,9 @@ class CameraAdaptor(NodeAdaptor[TCamera, TNative]):
     @abstractmethod
     def _snx_set_type(self, arg: model.CameraType, /) -> None: ...
     @abstractmethod
-    def _snx_set_zoom(self, arg: float, /) -> None: ...
-    @abstractmethod
-    def _snx_set_center(self, arg: tuple[float, ...], /) -> None: ...
-    @abstractmethod
     def _snx_zoom_to_fit(self, arg: float, /) -> None: ...
+    @abstractmethod
+    def _snx_set_projection(self, arg: model.Transform, /) -> None: ...
 
 
 class ImageAdaptor(NodeAdaptor[TImage, TNative]):

src/scenex/adaptors/_pygfx/_camera.py

@@ -13,6 +13,7 @@
 
 if TYPE_CHECKING:
     from scenex import model
+    from scenex.model import Transform
 
 logger = logging.getLogger("scenex.adaptors.pygfx")
 
@@ -35,12 +36,6 @@ def __init__(self, camera: model.Camera, **backend_kwargs: Any) -> None:
 
         self._pygfx_node.local.scale_y = -1  # don't think this is working...
 
-    def _snx_set_zoom(self, zoom: float) -> None:
-        logger.warning("'Camera._snx_set_zoom' not implemented for pygfx")
-
-    def _snx_set_center(self, arg: tuple[float, ...]) -> None:
-        logger.warning("'Camera._snx_set_center' not implemented for pygfx")
-
     def _snx_set_type(self, arg: model.CameraType) -> None:
         logger.warning("'Camera._snx_set_type' not implemented for pygfx")
 
@@ -60,6 +55,9 @@ def set_viewport(self, viewport: pygfx.Viewport) -> None:
         # and should perhaps be moved to the View Adaptor
         self.pygfx_controller.add_default_event_handlers(viewport, self._pygfx_node)
 
+    def _snx_set_projection(self, arg: Transform) -> None:
+        self._pygfx_node.projection_matrix = arg.root  # pyright: ignore[reportAttributeAccessIssue]
+
     def _snx_zoom_to_fit(self, margin: float) -> None:
         # reset camera to fit all objects
         if not (scene := self._camera_model.parent):
@@ -79,3 +77,8 @@ def _snx_zoom_to_fit(self, margin: float) -> None:
             cam.width = width
             cam.height = height
         cam.zoom = 1 - margin
+        # FIXME: Pyright
+        self._camera_model.transform = cam.local.matrix.T  # pyright: ignore[reportAttributeAccessIssue]
+        # HACK: Ideally, we'd use `cam.projection_matrix`, but it's a cached
+        # property that doesn't get recomputed.
+        self._camera_model.projection = cam._update_projection_matrix()  # pyright: ignore[reportAttributeAccessIssue]

src/scenex/adaptors/_pygfx/_volume.py

@@ -67,6 +67,7 @@ def _snx_set_render_mode(
             kwargs["interpolation"] = interpolation
         elif self._material is not None:
             kwargs["interpolation"] = self._material.interpolation
+            kwargs["clim"] = self._material.clim
 
         if data == "mip":
             self._material = pygfx.VolumeMipMaterial(**kwargs)

src/scenex/adaptors/_vispy/_camera.py

@@ -2,16 +2,16 @@
 
 from typing import TYPE_CHECKING, Any
 
-import numpy as np
+import vispy.geometry
 import vispy.scene
 
 from scenex.adaptors._base import CameraAdaptor
+from scenex.model import Transform
 
 from ._node import Node
 
 if TYPE_CHECKING:
     from scenex import model
-    from scenex.model import Transform
 
 
 class Camera(Node, CameraAdaptor):
@@ -21,37 +21,72 @@ class Camera(Node, CameraAdaptor):
 
     def __init__(self, camera: model.Camera, **backend_kwargs: Any) -> None:
         self._camera_model = camera
-        if camera.type == "panzoom":
-            self._vispy_node = vispy.scene.PanZoomCamera()
-            self._vispy_node.interactive = True
-        elif camera.type == "perspective":
-            # TODO: These settings were copied from the pygfx camera.
-            # Unify these values?
-            self._vispy_node = vispy.scene.ArcballCamera(70)
 
-        self._snx_zoom_to_fit(0.1)
-
-    def _snx_set_zoom(self, zoom: float) -> None:
-        self._vispy_node.zoom_factor = zoom
-
-    def _snx_set_center(self, arg: tuple[float, ...]) -> None:
-        self._vispy_node.center = arg
+        # The camera model contains:
+        # A projection transform, mapping local space to NDC
+        # A parent transform, mapping parent space to local space
+        #
+        # TODO: We may need a utility to get a transform mapping world space to local
+        # space.
+        #
+        # The BaseCamera.transform field should map world space to canvas position.
+        #
+        # To construct this transform from our camera model, we need:
+        # 1) A transform from world space to local space:
+        #   Note that this is usually the inverse of the model's transform matrix
+        self._transform = Transform()
+        # 2) A transform from local space to NDC:
+        self._projection = Transform()
+        # 3) A transform from NDC to canvas position:
+        self._from_NDC = Transform()
+
+        self._vispy_node = vispy.scene.BaseCamera()
+        # FIXME: Compared to pygfx, the y-axis appears inverted.
+        # The line below does not help...
+        # self._vispy_node.flip = (False, True, False)
+
+    def _set_view(self, view: vispy.scene.ViewBox) -> None:
+        # map [-1, -1] to [0, 0]
+        # map [1, 1] to [w, h]
+        w, h = view.size
+        self._from_NDC = Transform().translated((1, 1)).scaled((w / 2, h / 2, 1))
+
+        self._update_vispy_node_tform()
 
     def _snx_set_type(self, arg: model.CameraType) -> None:
         raise NotImplementedError()
 
     def _snx_set_transform(self, arg: Transform) -> None:
-        if isinstance(self._vispy_node, vispy.scene.PanZoomCamera):
-            self._vispy_node.tf_mat = vispy.scene.transforms.MatrixTransform(
-                np.asarray(arg)
-            )
-        else:
-            super()._snx_set_transform(arg)
+        # Note that the scenex transform is inverted here.
+        # Scenex transforms map local coordinates into parent coordinates,
+        # but our vispy node's transform must go the opposite way, from world
+        # coordinates into parent coordinates.
+        #
+        # FIXME: Note the discrepancy between world and parent coordinates. World
+        # coordinates are needed for the vispy transform node, but the current transform
+        # only converts local to parent space. This will likely be a source of bugs for
+        # more complicated scenes. There's also a TODO above about fixing this.
+        self._transform = arg.inv()
+        self._update_vispy_node_tform()
+
+    def _snx_set_projection(self, arg: Transform) -> None:
+        self._projection = arg
+        # Have to recompute the vispy transform offset if the projection changed
+        self._snx_set_transform(self._camera_model.transform)
+        # FIXME this call is redundant since _snx_set_transform does it, but it's
+        # worth remembering that this needs to happen.
+        self._update_vispy_node_tform()
+
+    def _update_vispy_node_tform(self) -> None:
+        mat = self._transform @ self._projection.T @ self._from_NDC
+        self._vispy_node.transform = vispy.scene.transforms.MatrixTransform(mat.root)
+        self._vispy_node.view_changed()
 
     def _view_size(self) -> tuple[float, float] | None:
         """Return the size of first parent viewbox in pixels."""
         raise NotImplementedError
 
     def _snx_zoom_to_fit(self, margin: float) -> None:
         # reset camera to fit all objects
-        self._vispy_node.set_range()
+        # FIXME: Implement this code in the model
+        self._vispy_node.set_range(margin=margin)

src/scenex/adaptors/_vispy/_canvas.py

@@ -52,7 +52,12 @@ def _draw(self) -> None:
         self._canvas.update()
 
     def _snx_add_view(self, view: model.View) -> None:
-        self._grid.add_widget(get_adaptor(view)._snx_get_native())
+        adaptor = get_adaptor(view)
+        self._grid.add_widget(adaptor._snx_get_native())
+        # HACK: Update view size by passing the existing camera
+        self._grid._prepare_draw(adaptor._snx_get_native())
+        cam_adaptor = get_adaptor(view.camera)
+        cam_adaptor._set_view(adaptor._vispy_viewbox)  # type: ignore
 
     def _snx_set_width(self, arg: int) -> None:
         self._canvas.size = (self._canvas.size[0], arg)

src/scenex/adaptors/_vispy/_view.py

@@ -85,6 +85,9 @@ def _snx_set_camera(self, cam: model.Camera) -> None:
         self._vispy_camera = self._cam_adaptor._vispy_node
         if hasattr(self, "_vispy_viewbox"):
             self._vispy_viewbox.camera = self._vispy_camera
+            # Vispy camera transforms need knowledge of viewbox
+            # (specifically, its size)
+            self._cam_adaptor._set_view(self._vispy_viewbox)
 
     def _draw(self) -> None:
         self._vispy_viewbox.update()

src/scenex/model/_nodes/camera.py

@@ -4,6 +4,8 @@
 
 from pydantic import Field
 
+from scenex.model._transform import Transform
+
 from .node import Node
 
 CameraType = Literal["panzoom", "perspective"]
@@ -17,6 +19,13 @@ class Camera(Node):
 
     The camera lives in, and is a child of, a scene graph.  It defines the view
     transformation for the scene, mapping it onto a 2D surface.
+
+    Cameras have two different Transforms. Like all Nodes, it has a transform
+    `transform`, describing its location in the world. Its other transform,
+    `projection`, describes how 2D normalized device coordinates
+    {(x, y) | x in [-1, 1], y in [-1, 1]} map to a ray in 3D world space. The inner
+    product of these matrices can convert a 2D canvas position to a 3D ray eminating
+    from the camera node into the world.
     """
 
     node_type: Literal["camera"] = "camera"
@@ -27,7 +36,8 @@ class Camera(Node):
         description="Whether the camera responds to user interaction, "
         "such as mouse and keyboard events.",
     )
-    zoom: float = Field(default=1.0, description="Zoom factor of the camera.")
-    center: Position = Field(
-        default=(0, 0, 0), description="Center position of the view."
+    # FIXME: Default should be explained. And z-scale should probably be -1
+    projection: Transform = Field(
+        default_factory=Transform,
+        description="Describes how 3D points are mapped to a 2D canvas",
     )

src/scenex/model/_transform.py

@@ -3,7 +3,7 @@
 import functools
 import math
 from functools import reduce
-from typing import TYPE_CHECKING, Any, ClassVar, cast
+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, cast
 
 import numpy as np
 from pydantic import ConfigDict, Field, RootModel
@@ -89,8 +89,8 @@ def _serialize(val: np.ndarray) -> list | None:
 class Transform(RootModel):
     """A 4x4 transformation matrix placing a 3D object in 3D space."""
 
-    root: Matrix3D = Field(
-        default_factory=lambda: np.eye(4),  # type: ignore
+    root: Annotated[np.ndarray, Matrix3D] = Field(
+        default_factory=lambda: np.eye(4),
         description="4x4 Transformation matrix.",
     )
 
@@ -116,7 +116,7 @@ def __matmul__(self, other: Transform | ArrayLike) -> Transform:
         """Return the dot product of this transform with another."""
         if isinstance(other, Transform):
             other = other.root
-        return Transform(self.root @ other)  # type: ignore
+        return Transform(self.root @ other)
 
     def dot(self, other: Transform | ArrayLike) -> Transform:
         """Return the dot product of this transform with another."""
@@ -131,7 +131,7 @@ def T(self) -> Transform:
 
     def inv(self) -> Transform:
         """Return the inverse of the transform."""
-        return Transform(np.linalg.inv(self.root))  # type: ignore
+        return Transform(np.linalg.inv(self.root))
 
     def translated(self, pos: ArrayLike) -> Transform:
         """Return new transform, translated by pos.