Corrupted / garbled camera images when using Gymnasium SyncVectorEnv with robosuite

[ShahRutav]

System Info

robosuite==1.5.2
numpy==1.23.5
gymnasium==1.2.3

Information

import os
import numpy as np
import gymnasium as gym
import matplotlib.pyplot as plt
from gymnasium.vector import SyncVectorEnv

import robosuite as suite
from robosuite.wrappers import GymWrapper

def make_env(seed: int, camera_names: list[str]):
    def _thunk():
        # Create a robosuite env (example: Lift)
        rs_env = suite.make(
            env_name="Lift",
            robots="Panda",
            has_renderer=False,          
            has_offscreen_renderer=True,
            use_camera_obs=True, 
            camera_names=camera_names,
            horizon=400,
            control_freq=20,
            use_object_obs=False,
            camera_heights=128,
            camera_widths=128,
        )

        # Wrap to Gym-style API
        env = GymWrapper(rs_env, flatten_obs=False)  # returns obs, reward, done, info (classic gym style)
        env.metadata = {} # hack since robosuite sets metadata to None
        env.reset(seed=seed)
        return env
    return _thunk

# Build a 2-env vector
n_envs = 2
camera_names = ["agentview", "robot0_eye_in_hand"]
venv = SyncVectorEnv([make_env(i, camera_names) for i in range(n_envs)], autoreset_mode=gym.vector.AutoresetMode.SAME_STEP)

obs, info = venv.reset()

for i in range(10):
    actions = venv.action_space.sample()  # samples a batch of 2 actions automatically for vector envs
    obs, rewards, terminated, truncated, infos = venv.step(actions)

    image = np.concatenate([obs[camera_names[0] + "_image"][j] for j in range(n_envs)], axis=1)
    # save each image to a file in /tmp/
    image = image.astype(np.uint8)
    plt.imshow(image)
    filename = f"/tmp/image_{i}.png"
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    plt.savefig(filename)
    print(f"Saved image to {filename}")
    plt.close()

Reproduction

When I wrap robosuite envs with GymWrapper(flatten_obs=False) and vectorize them using Gymnasium’s SyncVectorEnv, the image observations returned in obs["<camera>_image"] are corrupted.

Expected behavior

Image observations from each env should be valid RGB frames

[ShahRutav]

Example agentview image (left is garbage, right is eye_in_hand)

[ShahRutav]

Setting hard_reset=False resolves the issue for robosuite Lift environment. However, the issue persists for RoboCasa environments.

.
Left: agentview image from the first environment. Right: agentview image from the second.

import os
import numpy as np
import gymnasium as gym
import matplotlib.pyplot as plt
from gymnasium.vector import SyncVectorEnv

import robocasa
import robosuite as suite
from robosuite.wrappers import GymWrapper

def make_env(seed: int, camera_names: list[str]):
    def _thunk():
        # Create a robosuite env (example: Lift)
        rs_env = suite.make(
            env_name="TurnOnStove",
            robots="PandaOmron",
            has_renderer=False,          
            has_offscreen_renderer=True,
            use_camera_obs=True, 
            camera_names=camera_names,
            horizon=400,
            control_freq=20,
            use_object_obs=False,
            camera_heights=128,
            camera_widths=128,
            hard_reset=False,
        )

        # Wrap to Gym-style API
        env = GymWrapper(rs_env, flatten_obs=False)  # returns obs, reward, done, info (classic gym style)
        env.metadata = {} # hack since robosuite sets metadata to None
        env.reset(seed=seed)
        return env
    return _thunk

# Build a 2-env vector
n_envs = 2
camera_names = ["robot0_agentview_center", "robot0_eye_in_hand"]
# camera_names = ["robot0_robotview", "robot0_eye_in_hand"]
venv = SyncVectorEnv([make_env(i, camera_names) for i in range(n_envs)], autoreset_mode=gym.vector.AutoresetMode.SAME_STEP)

obs, info = venv.reset()

for i in range(10):
    actions = venv.action_space.sample()  # samples a batch of 2 actions automatically for vector envs
    obs, rewards, terminated, truncated, infos = venv.step(actions)

    filename = f"/tmp/image_{i}.png"
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    image = np.concatenate([obs[camera_names[0] + "_image"][j] for j in range(n_envs)], axis=1)
    image = image[::-1]
    image = image.astype(np.uint8)
    plt.imshow(image); plt.axis("off"); plt.tight_layout()
    plt.savefig(filename)
    plt.close()
    print(f"Saved image to {filename}")

cc: @snasiriany @Abhiram824

[Abhiram824]

can you try using this branch for robosuite: https://github.com/ARISE-Initiative/robosuite/tree/syncvectorenv-fix. I still need to do some testing, but let me know if it works

[ShahRutav]

Thanks @Abhiram824! A combination of hard_reset=False and the fix you mentioned resolves the issue. Let's merge this into the master branch?

[sean1295]

can you try using this branch for robosuite: https://github.com/ARISE-Initiative/robosuite/tree/syncvectorenv-fix. I still need to do some testing, but let me know if it works

I also ran into this issue, but without using vectorized env. Adding this line fixes the issue. Thanks!

[Abhiram824]

Glad it worked, let me open a PR