Add Kit-less OVRTX rendering with Newton and OvPhysX backends

Author: dengyuchenkit

compass/utils/renderer_utils.py

@@ -24,11 +24,13 @@
 # pylint: disable=import-outside-toplevel
 
 ###############################
-# Isaac Sim RTX Renderer Setup
+# Camera Renderer Setup
 ###############################
 __all__ = [
+    "apply_camera_renderer_settings",
     "apply_isaac_rtx_camera_renderer_settings",
     "apply_nurec_spg_kit_args",
+    "configure_renderer_runtime",
     "configure_nurec_isaacsim_rtx_viewport",
 ]
 
@@ -65,6 +67,66 @@
 
 
 # Public API
+def configure_renderer_runtime(args):
+    """Configure renderer startup before importing the simulation environment."""
+    physics_backend = getattr(args, "physics_backend", None)
+    if physics_backend is None:
+        physics_backend = "physx" if args.camera_renderer == "isaac_rtx" else "newton"
+    compatible_backends = {
+        "isaac_rtx": ("physx",),
+        "ovrtx": ("newton", "ovphysx"),
+    }
+    if physics_backend not in compatible_backends[args.camera_renderer]:
+        raise ValueError(
+            f"--camera-renderer {args.camera_renderer} does not support "
+            f"--physics-backend {physics_backend}; choose "
+            f"{', '.join(compatible_backends[args.camera_renderer])}.")
+    args.physics_backend = physics_backend
+
+    if args.camera_renderer == "isaac_rtx":
+        apply_nurec_spg_kit_args(args)
+        return
+
+    from compass.utils.visualizer_utils import requested_visualizers
+
+    if "kit" in requested_visualizers(args):
+        raise ValueError(
+            "--camera-renderer ovrtx requires a Kit-less process; remove --visualizer kit.")
+    livestream = getattr(args, "livestream", -1)
+    if livestream is None or livestream < 0:
+        livestream = int(os.environ.get("LIVESTREAM", "0"))
+    if livestream or getattr(args, "xr", False):
+        raise ValueError("--camera-renderer ovrtx does not support Kit livestreaming or XR.")
+    if getattr(args, "kit_args", None) or getattr(args, "experience", None):
+        raise ValueError("--camera-renderer ovrtx does not use --kit_args or --experience.")
+
+    args.spg_runtime = uses_nurec_spg_runtime(args)
+    # AUTO_ANY ISP requests HDR even before a scene's PPISP shader is discovered.
+    # Match ppisp_camera_ovrtx.py; retain explicit user overrides.
+    os.environ.setdefault("OVRTX_rtx_rtpt_gaussian_skipTonemapping_enabled", "0")
+
+
+def apply_camera_renderer_settings(env_cfg, args):
+    """Select the camera renderer and its compatible physics backend."""
+    if args.camera_renderer == "isaac_rtx":
+        apply_isaac_rtx_camera_renderer_settings(env_cfg, True, args.spg_runtime)
+        return
+
+    from isaaclab_ov.renderers import OVRTXRendererCfg
+
+    # Both choices run without Isaac Sim's Kit-based PhysX runtime.
+    if args.physics_backend == "ovphysx":
+        from isaaclab_ov.physics import OvPhysxCfg
+
+        env_cfg.sim.physics = OvPhysxCfg()
+    else:
+        from isaaclab_newton.physics import NewtonCfg
+        from isaaclab_newton.physics.mjwarp_manager_cfg import MJWarpSolverCfg
+
+        env_cfg.sim.physics = NewtonCfg(solver_cfg=MJWarpSolverCfg(), num_substeps=1)
+    env_cfg.scene.camera.renderer_cfg = OVRTXRendererCfg()
+
+
 def apply_isaac_rtx_camera_renderer_settings(env_cfg, isaac_rtx, spg_runtime):
     """Apply Isaac RTX camera renderer settings when the Isaac RTX renderer is used."""
     if not isaac_rtx:

docs/handbook/workflows/nurec_real2sim.md

@@ -134,6 +134,64 @@ This command uses the default `particle_spg-runtime.usdz`. Use
 `--nurec-usd-file <filename>` to choose another USD from the selected scene
 folder.
 
+#### OVRTX camera rendering
+
+Add `--camera-renderer ovrtx` to use OVRTX for onboard RGB/depth cameras.
+By default this selects Newton MJWarp physics and runs without Kit, following Isaac Lab's
+`scripts/demos/sensors/ppisp_camera_ovrtx.py` example. Isaac RTX with PhysX remains
+the default. Switching to OVRTX also changes the physics backend; validate
+locomotion policies with the selected backend before training.
+
+Use a Kit-less Isaac Lab Python environment with the `ovrtx` extra installed
+(for example, `uv sync --extra ovrtx` in the Isaac Lab checkout), plus the
+COMPASS dependencies and `mobility_es` extension. The Docker Python wrapper
+described above launches Isaac Sim's Python; use the Kit-less environment for
+this command:
+
+```bash
+python run.py \
+    -c configs/train_config_real2sim.gin \
+    -o <output_dir> \
+    -b ./assets/x_mobility.ckpt \
+    --embodiment carter \
+    --nurec-scene nova_carter-galileo \
+    --num_envs 4 \
+    --camera-renderer ovrtx \
+    --physics-backend newton \
+    --visualizer none
+```
+
+To use standalone PhysX instead, replace `--physics-backend newton` with
+`--physics-backend ovphysx`. Install both runtime extras in the Kit-less Isaac Lab
+environment (`uv sync --extra ovrtx --extra ovphysx`) before running it.
+OvPhysX is experimental; this selects `OvPhysxCfg`, rather than Isaac Sim's
+Kit-based `PhysxCfg`. It uses its own backend defaults and USD physics-scene
+settings, so do not assume identical dynamics to the Isaac Sim path.
+
+For an OvPhysX smoke run on `nova_carter-galileo`, add
+`--nurec-usd-file volume.usdz`. This combination has been checked locally with
+Carter at 512x320 resolution, including stepping and reset. The default
+`particle_spg-runtime.usdz` has not completed local OvPhysX validation: Isaac
+Lab 3.0 EA serializes the full physics input stage as USDA text, and this large
+particle scene showed substantial host-memory growth during initialization.
+
+Supported combinations are:
+
+| Camera renderer | Physics backend |
+|---|---|
+| `isaac_rtx` | `physx` (default) |
+| `ovrtx` | `newton` (default, MJWarp solver) or `ovphysx` |
+
+Incompatible combinations are rejected before starting either runtime.
+
+Automatic PPISP discovery remains enabled. COMPASS sets
+`OVRTX_rtx_rtpt_gaussian_skipTonemapping_enabled=0` before loading OVRTX unless
+you explicitly set it in the environment. OVRTX does not accept the Kit
+visualizer, livestreaming, XR, `--kit_args`, or `--experience`. Robot-camera
+videos and debug grids remain available with `--video`.
+
+#### Visualization and common options
+
 | Kit viewport | Robot camera |
 |---|---|
 | <img src="images/nurec_particle_spg_runtime_kit_viewport.jpg" alt="Particle SPG-runtime Kit viewport" width="320"> | <img src="images/nurec_particle_spg_runtime_camera_grid.png" alt="Particle SPG-runtime robot-camera debug grid" width="800"> |

run.py

@@ -23,8 +23,8 @@
 
 from compass.utils.nurec_utils import PARTICLE_SPG_RUNTIME_USD_FILE
 from compass.utils.renderer_utils import (
-    apply_isaac_rtx_camera_renderer_settings,
-    apply_nurec_spg_kit_args,
+    apply_camera_renderer_settings,
+    configure_renderer_runtime,
     configure_nurec_isaacsim_rtx_viewport,
 )
 from compass.utils.visualizer_utils import (
@@ -35,6 +35,19 @@
 
 # add argparse arguments
 parser = argparse.ArgumentParser(description="COMPASS Mobility Generalist.")
+parser.add_argument(
+    "--camera-renderer",
+    choices=["isaac_rtx", "ovrtx"],
+    default="isaac_rtx",
+    help="Camera renderer. OVRTX runs without Kit; select physics with --physics-backend.",
+)
+parser.add_argument(
+    "--physics-backend",
+    choices=["physx", "newton", "ovphysx"],
+    default=None,
+    help="Physics backend. Defaults to physx for Isaac RTX or newton (MJWarp) for OVRTX. "
+    "Standalone ovphysx requires --camera-renderer ovrtx.",
+)
 parser.add_argument(
     "--config-files",
     "-c",
@@ -142,7 +155,7 @@
     "--spg-runtime",
     action="store_true",
     default=False,
-    help="Force SPG runtime Kit args before Isaac Sim starts. "
+    help="Force SPG runtime renderer settings before simulation starts. "
     f"This is automatic for {PARTICLE_SPG_RUNTIME_USD_FILE} "
     "when --nurec-scene is set.",
 )
@@ -187,7 +200,10 @@
     args_cli.environment = args_cli.nurec_scene
 elif args_cli.nurec_omap_file is not None:
     parser.error("--nurec-omap-file requires --nurec-scene.")
-apply_nurec_spg_kit_args(args_cli)
+try:
+    configure_renderer_runtime(args_cli)
+except ValueError as exc:
+    parser.error(str(exc))
 
 if args_cli.video:
     # Load the FFmpeg-enabled wheel before Kit prepends its bundled OpenCV to sys.path.
@@ -198,9 +214,25 @@
             "--video requires FFmpeg-enabled OpenCV. Install requirements.txt and recreate "
             f"the Docker container after rebuilding the image. Loaded OpenCV: {cv2.__file__}")
 
-# launch omniverse app
-app_launcher = AppLauncher(args_cli, enable_cameras=True)
-simulation_app = app_launcher.app
+# OVRTX must never construct AppLauncher, even for headless runs.
+app_launcher = None
+simulation_app = None
+if args_cli.camera_renderer == "isaac_rtx":
+    app_launcher = AppLauncher(args_cli, enable_cameras=True)
+    simulation_app = app_launcher.app
+else:
+    try:
+        import ovrtx
+    except ModuleNotFoundError as exc:
+        raise ModuleNotFoundError(
+            "OVRTX requires Isaac Lab's optional ovrtx dependencies. "
+            "Install the ovrtx extra and run with the Kit-less Python environment.") from exc
+    ovrtx.register_schema_paths()
+    AppLauncher.sync_visualizer_cli_settings_to_carb({
+        **vars(args_cli),
+        "visualizer_disable_all":
+            getattr(args_cli, "visualizer_explicit", False) and not args_cli.visualizer,
+    })
 
 import gin
 import torch
@@ -302,14 +334,12 @@ def run(
     num_rerenders_on_reset=None,
 ):
 
-    # Multi-GPU distributed setup. With `--distributed`, AppLauncher (already invoked
-    # at module load) reads LOCAL_RANK / RANK / WORLD_SIZE from torchrun's env, sets
-    # physics/active GPU per rank, and limits CPU threads. We still need to call
-    # init_process_group ourselves before any cross-rank op (param broadcast in
-    # ResidualPPOTrainer.__init__, gradient all-reduce in PPO.update).
+    # Kit-less runs read torchrun ranks directly because AppLauncher is skipped.
     if args_cli.distributed:
-        local_rank = app_launcher.local_rank
-        global_rank = app_launcher.global_rank
+        local_rank = (app_launcher.local_rank if app_launcher is not None else int(
+            os.environ.get("LOCAL_RANK", "0")))
+        global_rank = (app_launcher.global_rank if app_launcher is not None else int(
+            os.environ.get("RANK", "0")))
         # Pin PyTorch's current CUDA device to this rank's GPU BEFORE
         # init_process_group / any object-collective. NCCL's object
         # collectives (dist.all_gather_object in _save_episode_logs)
@@ -326,7 +356,8 @@ def run(
     else:
         local_rank = 0
         global_rank = 0
-        device = "cuda" if torch.cuda.is_available() else "cpu"
+        device = (args_cli.device if args_cli.camera_renderer == "ovrtx" else
+                  "cuda" if torch.cuda.is_available() else "cpu")
         is_rank_zero = True
 
     # Setup logger. Only rank 0 writes TensorBoard / W&B / artifacts; other ranks get
@@ -398,21 +429,15 @@ def run(
 
     requested_viz = requested_visualizers(args_cli)
     configure_visualizers(env_cfg, requested_viz)
-    apply_isaac_rtx_camera_renderer_settings(
-        env_cfg,
-        isaac_rtx=True,
-        spg_runtime=args_cli.spg_runtime,
-    )
+    apply_camera_renderer_settings(env_cfg, args_cli)
 
     # Setup seed. Per-rank offset diversifies env initial conditions across GPUs so
     # rollouts collected by each rank explore different states (matches Isaac Lab's
     # rsl_rl reference pattern).
     env_cfg.seed = seed + global_rank
 
-    # Pin PhysX + Isaac Sim's render device to this rank's GPU. Without this every
-    # rank's env_cfg.sim.device defaults to cuda:0 and all 8 sims pile onto a single
-    # GPU (caught with a Vulkan OOM during material loading on the first attempt).
-    if args_cli.distributed:
+    # Keep physics and rendering on the policy device for Kit-less and distributed runs.
+    if args_cli.distributed or args_cli.camera_renderer == "ovrtx":
         env_cfg.sim.device = device
 
     # Set collision distances and max resample trial from gin config
@@ -513,6 +538,7 @@ def run(
     logger.log_config(gin_config_to_dictionary(gin.config._OPERATIVE_CONFIG))
 
     logger.close()
+    env.close()
 
 
 def main():
@@ -540,6 +566,8 @@ def main():
 
 if __name__ == "__main__":
     # Run the main function.
-    main()
-    # Close the sim app.
-    simulation_app.close()
+    try:
+        main()
+    finally:
+        if simulation_app is not None:
+            simulation_app.close()