diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..700b742e8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,182 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# ROS +build/ +devel/ +install/ +log/ + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +dist/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +logs/ +.vscode/ + +!examples/imgs/*.md + +src/examples/*-demo +artifact_database.pkl + +imgui.ini + +src/rai_bench/rai_bench/experiments +src/rai_interfaces + +# Demo assets +demo_assets/ diff --git a/docker/Dockerfile b/docker/Dockerfile index 494092677..696eb9ebf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,27 +13,42 @@ # limitations under the License. ARG ROS_DISTRO=jazzy - FROM osrf/ros:${ROS_DISTRO}-desktop-full + +ARG DEPENDENCIES=core_only ENV DEBIAN_FRONTEND=noninteractive +# Check whether the $DEPENDENCIES ARG has a valid value +RUN /bin/bash -c 'if [ "${DEPENDENCIES}" != "core_only" ] && [ "${DEPENDENCIES}" != "all_groups" ]; then \ + echo Error: invalid DEPENDENCIES value. Valid values "core_only", "all_groups" && exit 1; \ +fi' + + # Install dependencies -RUN apt-get update && apt-get install -y \ +RUN apt update && apt install -y \ python3 \ python3-pip \ git \ - wget + wget \ + zip # Install Poetry -RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.4 +RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3 ENV PATH="/root/.local/bin:$PATH" -# Clone and setup RAI +# Copy the RAI repository +COPY . /rai WORKDIR /rai -RUN git clone https://github.com/RobotecAI/rai.git . -# Install Python dependencies with Poetry -RUN poetry install --with nomad,openset +# Set up the environment to use Fast DDS +ENV FASTRTPS_DEFAULT_PROFILES_FILE=/rai/docker/fastrtps_config.xml +ENV RMW_IMPLEMENTATION=rmw_fastrtps_cpp + +RUN /bin/bash -c 'if [ "${DEPENDENCIES}" = "core_only" ]; then \ + poetry install; \ +else \ + poetry install --all-groups; \ +fi' # Install ROS dependencies RUN /bin/bash -c '. /opt/ros/${ROS_DISTRO}/setup.bash && \ diff --git a/docker/fastrtps_config.xml b/docker/fastrtps_config.xml new file mode 100644 index 000000000..288776e1b --- /dev/null +++ b/docker/fastrtps_config.xml @@ -0,0 +1,17 @@ + + + + + CustomUdpTransport + UDPv4 + + + + + + CustomUdpTransport + + false + + + diff --git a/docs/demos/manipulation.md b/docs/demos/manipulation.md index 02ecd6a19..3e2f280a0 100644 --- a/docs/demos/manipulation.md +++ b/docs/demos/manipulation.md @@ -18,6 +18,10 @@ manipulation techniques. Make sure ROS 2 is sourced. (e.g. `source /opt/ros/humble/setup.bash`) +### Local Setup + +#### Setting up the demo + 1. Follow the RAI setup instructions in the [quick setup guide](../setup/install.md#setting-up-developer-environment). 2. Download additional dependencies: @@ -39,7 +43,7 @@ manipulation techniques. colcon build --symlink-install ``` -## Running the Demo +#### Running the demo !!! note "Remain in sourced shell" @@ -75,6 +79,110 @@ manipulation techniques. To change camera in the simulation use 1-7 keys on your keyboard once it's window is focused. +### Docker Setup + +!!! note "ROS 2 required" + + The docker setup requires a working Humble or Jazzy ROS 2 installation on the host machine. Make sure that ROS 2 is sourced on the host machine and the `ROS_DOMAIN_ID` environment variable is set to the same value as in the [Docker setup](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional) + +!!! warning "ROS 2 distributions" + + It is highly recommended that ROS 2 distribution on the host machine matches the ROS 2 distribution of the docker container. A distribution version mismatch may result in the demo not working correctly. + +#### 1. Setting up the demo + +1. Set up docker as outlined in the [docker setup guide](../setup/setup_docker.md). During the setup, build the docker image with all dependencies (i.e., use the `--build-arg DEPENDENCIES=all_groups` argument) + and configure communication between the container and the host ([link](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional)). + +2. On the host machine, download the latest binary release for the Robotic Arm Demo: + + ```shell + ./scripts/download_demo.sh manipulation + ``` + +3. Run the docker container (if not already running): + + ```shell + docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:jazzy # or rai:humble + ``` + + !!! tip "NVIDIA GPU acceleration" + + If the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) is set up on your host machine, you can use the GPU within the RAI docker container for faster inference by adding the `--gpus all` option: + + ```shell + docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID --gpus all -it rai:jazzy # or rai:humble + ``` + + Sometimes, passing GPUs to the docker container may result in an error: + + ```shell + docker: Error response from daemon: could not select device driver "" with capabilities: [[gpu]]. + ``` + + Restarting the docker service should resolve this error: + + ```shell + sudo systemctl restart docker + ``` + +4. (Inside the container shell) Download additional ROS 2 dependencies: + + ```shell + vcs import < demos.repos + rosdep install --from-paths src/examples/rai-manipulation-demo/ros2_ws/src --ignore-src -r -y + ``` + +5. (Inside the container shell) Build the ROS 2 workspace: + + ```shell + source /opt/ros/${ROS_DISTRO}/setup.bash + colcon build --symlink-install + ``` + +6. (Inside the docker container) By default, RAI uses OpenAI as the vendor. Thus, it is necessary + to set the `$OPENAI_API_KEY` environmental variable. The command below may be utilized to set + the variable and add it to the container's `.bashrc` file: + + ```shell + export OPENAI_API_KEY=YOUR_OPEN_AI_API_KEY + echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> ~/.bashrc + ``` + + !!! note AI vendor change + + The default vendor can be changed to a different provider via the [RAI configuration tool](../setup/install.md#15-configure-rai) + +#### 2. Running the demo + +!!! note Source the setup shell + + Ensure ROS 2 is sourced on the host machine and the `ROS_DOMAIN_ID` environment variable is set to the same value as in the [Docker setup](../setup/setup_docker.md#2-set-up-communications-between-docker-and-host-optional). Ensure that every command inside the docker container is run in a sourced shell using `source setup_shell.sh`. + +1. Launch the Robotic Arm Visualization on the host machine: + + ```shell + ./demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher + ``` + +2. (Inside the container shell) Launch the Robotic Arm Demo script inside of the docker container: + + ```shell + ros2 launch examples/manipulation-demo.launch.py + ``` + +3. (Inside the container shell) Open a new terminal for the docker container (e.g., `docker exec -it CONTAINER_ID /bin/bash`) and launch the streamlit interface: + + ```shell + streamlit run examples/manipulation-demo-streamlit.py + ``` + + Alternatively, run the simpler command-line version: + + ```shell + python examples/manipulation-demo.py + ``` + ## How it works The manipulation demo utilizes several components: diff --git a/docs/setup/setup_docker.md b/docs/setup/setup_docker.md index ef7b0ebef..c186d5800 100644 --- a/docs/setup/setup_docker.md +++ b/docs/setup/setup_docker.md @@ -7,48 +7,105 @@ ## 1. Build the docker image -Choose the docker image based on your preferred ROS 2 version. +Choose the docker image based on your preferred ROS 2 version. You may build the selected image with only the core dependencies or, alternatively, with all the additional modules. To build the docker image, you must clone the RAI repository: + +```bash +git clone https://github.com/RobotecAI/rai.git +cd rai +``` ### 1.1. Humble +Core dependencies only: + ```bash docker build -t rai:humble --build-arg ROS_DISTRO=humble -f docker/Dockerfile . ``` +All dependencies: + +```bash +docker build -t rai:humble --build-arg ROS_DISTRO=humble --build-arg DEPENDENCIES=all_groups -f docker/Dockerfile . +``` + ### 1.2. Jazzy +Core dependencies only: + ```bash docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy -f docker/Dockerfile . ``` -## 2. Run the docker container +All dependencies: + +```bash +docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy --build-arg DEPENDENCIES=all_groups -f docker/Dockerfile . +``` + +## 2. Set up communications between docker and host (Optional) !!! tip "ROS 2 communication" If you intend to run demos on the host machine, ensure the docker container can communicate with it. Test this by running the standard ROS 2 example with one node in docker and one on the host: - [link](https://docs.ros.org/en/jazzy/Installation/Ubuntu-Install-Debs.html#try-some-examples). If - topics are not visible or cannot be subscribed to, try using - [rmw_cyclone_dds](https://github.com/ros2/rmw_cyclonedds) instead of the default - rmw_fastrtps_cpp. + [link](https://docs.ros.org/en/jazzy/Installation/Ubuntu-Install-Debs.html#try-some-examples). + +!!! warning "ROS 2 distributions" + + It is highly recommended that ROS 2 distribution on the host machine matches the ROS 2 distribution of the docker container. A distribution version mismatch may result in the demos not working correctly. + +To allow the container to communicate with the host machine, configure the host environment as presented below: + +1. Source ROS 2 on the host machine: + + ```shell + source /opt/ros/jazzy/setup.bash # or humble + ``` + +2. If not configured, set the `ROS_DOMAIN_ID` environment variable to a domain ID between 0 and 101, inclusive. Example: + + ```shell + export ROS_DOMAIN_ID=99 + ``` + +3. Install the eProsima Fast DDS middleware (should come preinstalled with ROS 2): + + ```shell + sudo apt install ros-"${ROS_DISTRO}"-fastrtps + ``` + +4. Configure the DDS middleware using the `fastrtps_config.xml` file included in the RAI repository: + + ```shell + export FASTRTPS_DEFAULT_PROFILES_FILE=$(pwd)/docker/fastrtps_config.xml + + ``` + +5. Set the RMW to use eProsima Fast DDS: + + ```shell + export RMW_IMPLEMENTATION=rmw_fastrtps_cpp + ``` + +## 3. Run the docker container -### 2.1. Humble +### 3.1. Humble ```bash -docker run --net=host --ipc=host --pid=host -it rai:humble +docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:humble ``` -### 2.2. Jazzy +### 3.2. Jazzy ```bash -docker run --net=host --ipc=host --pid=host -it rai:jazzy +docker run --net=host --ipc=host --pid=host -e ROS_DOMAIN_ID=$ROS_DOMAIN_ID -it rai:jazzy ``` -## 3. Run the tests to confirm the setup +## 4. Run the tests to confirm the setup ```sh cd /rai source setup_shell.sh -poetry run pytest +poetry run pytest tests/{agents,messages,tools,types} ``` diff --git a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py index c143b4d13..62efc43cf 100644 --- a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py +++ b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/agents/base_vision_agent.py @@ -34,6 +34,7 @@ def __init__( super().__init__() self._weights_path = Path(weights_path) os.makedirs(self._weights_path, exist_ok=True) + self._is_weights_path_set = False self._init_weight_path() self.weight_path = self._weights_path self.ros2_connector = ROS2Connector(ros2_name, executor_type="single_threaded") @@ -43,9 +44,16 @@ def _init_weight_path(self): if self.WEIGHTS_FILENAME == "": raise ValueError("WEIGHTS_FILENAME is not set") - install_path = ( - self._weights_path / "vision" / "weights" / self.WEIGHTS_FILENAME - ) + # Ensure that the self._weights_path variable is set only once + # to prevent issues during weight re-downloading + if not self._is_weights_path_set: + install_path = ( + self._weights_path / "vision" / "weights" / self.WEIGHTS_FILENAME + ) + else: + install_path = self._weights_path + self._is_weights_path_set = True + # make sure the file exists if install_path.exists() and install_path.is_file(): self._weights_path = install_path diff --git a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py index 16c6fc2df..e8fa31245 100644 --- a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py +++ b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/tools/segmentation_tools.py @@ -345,13 +345,15 @@ def _run( conversion_ratio = 0.001 resolved = None - resolved = get_future_result(future) + # NOTE: Image processing by GroundingDino and Grounded SAM may take a significant amount + # of time, especially when performed on the CPU. Hence, timeout is set to 60 seconds + resolved = get_future_result(future, timeout_sec=60.0) assert resolved is not None future = self._call_gsam_node(camera_img_msg, resolved) ret = [] - resolved = get_future_result(future) + resolved = get_future_result(future, timeout_sec=60.0) if resolved is not None: for img_msg in resolved.masks: ret.append(convert_ros_img_to_base64(img_msg)) diff --git a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py index 949d18a2c..6bcb11008 100644 --- a/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py +++ b/src/rai_extensions/rai_open_set_vision/rai_open_set_vision/vision_markup/boxer.py @@ -17,6 +17,7 @@ from typing import Dict import cv2 +import torch from cv_bridge import CvBridge from groundingdino.util.inference import Model from rclpy.time import Time @@ -71,7 +72,8 @@ def __init__( if not use_cuda: self.model = Model(self.cfg_path, self.weight_path, device="cpu") else: - self.model = Model(self.cfg_path, self.weight_path) + device = "cuda" if torch.cuda.is_available() else "cpu" + self.model = Model(self.cfg_path, self.weight_path, device=device) self.bridge = CvBridge() def get_boxes(