From 05f95ec8573cfc8e2f23c3af8ed10d97a7c8dc7f Mon Sep 17 00:00:00 2001 From: ti Date: Wed, 23 Apr 2025 16:06:02 +0200 Subject: [PATCH 1/6] Add Dockerfile --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..36ef8a7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +FROM pytorch/pytorch:2.3.1-cuda11.8-cudnn8-runtime + +ARG user_name +ARG uid +ARG gid +ENV HOME=/root + +WORKDIR ${HOME} + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + vim zsh tmux wget curl htop jupyter python3 python3-pip libgl1-mesa-glx git sudo ssh libglib2.0-0 \ + tree \ + && apt-get -y autoclean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && apt-get clean \ + && apt-get update + +# Create a non-root user +RUN mkdir /app /logs /data +RUN groupadd -g ${gid} ${user_name} \ + && useradd -m -u ${uid} -g ${gid} ${user_name} \ + && chown -R ${uid}:${gid} /home +# +# Switch to the new user and set home directory as working directory +USER ${user_name} +ENV HOME=/home/${user_name} +WORKDIR ${HOME} + +# Install Oh My Zsh and plugins +RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" --unattended \ + && git clone https://github.com/zsh-users/zsh-autosuggestions ~/.oh-my-zsh/custom/plugins/zsh-autosuggestions \ + && git clone https://github.com/zsh-users/zsh-syntax-highlighting ~/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting \ + && sed -i 's/^plugins=(.*)$/plugins=(git zsh-autosuggestions zsh-syntax-highlighting)/' ~/.zshrc + +RUN echo "export PATH=$PATH:/home/${user_name}/.local/bin" >> /home/${user_name}/.zshrc + +# install pip packages +COPY ./conda/amadeusGPT.yml ${HOME}/amadeusGPT.yml +SHELL ["/bin/bash", "-lc"] +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda env create -f ${HOME}/amadeusGPT.yml && \ + conda activate amadeusgpt && \ + pip install --no-cache-dir --upgrade pip setuptools && \ + pip install --no-cache-dir gpustat nvitop pytest PySide6==6.3.1 streamlit networkx isort black git+https://github.com/DeepLabCut/DeepLabCut.git + +# USER ${user_name} +USER root +# ENV HOME=/root +WORKDIR /app + +CMD ["zsh"] +SHELL ["/bin/zsh", "-c"] \ No newline at end of file From 16a298135e66c89e66062f5e6cdf9e4aaf3be30f Mon Sep 17 00:00:00 2001 From: ti Date: Wed, 23 Apr 2025 16:06:12 +0200 Subject: [PATCH 2/6] Add Docker build and run commands to Makefile --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Makefile b/Makefile index 56f5a7b..42de1c2 100644 --- a/Makefile +++ b/Makefile @@ -2,3 +2,33 @@ export streamlit_app=True app: streamlit run amadeusgpt/app.py --server.fileWatcherType none --server.maxUploadSize 1000 + + +IMG_TAG := 0.1 +IMG_NAME := amadeusgpt +DOCKERFILE := Dockerfile + +BUILD_ARGS := \ + --build-arg uid=$(shell id -u) \ + --build-arg gid=$(shell id -g) \ + --build-arg user_name=$(shell id -un) +build: + docker build $(BUILD_ARGS) \ + -t $(IMG_NAME):$(IMG_TAG) -f $(DOCKERFILE) . + +# [USER: ADJUST VOLUMES] +# path to the local project +HOST_SRC := /home/$(shell id -un)/AmadeusGPT +# path to the project in the container +DOCKER_SRC := /home/$(shell id -un)/AmadeusGPT +# DOCKER_DATA := /mnt +VOLUMES := \ + --volume $(HOST_SRC):$(DOCKER_SRC) + +CONTAINER_TAG :=_test_v0.1 +CONTAINER_NAME := amadeusgpt_$(CONTAINER_TAG) # $(GPU) + +run: + docker run --shm-size=60G --gpus all -it --name $(CONTAINER_NAME) \ + $(VOLUMES) \ + $(IMG_NAME):$(IMG_TAG) tail -f /dev/null \ No newline at end of file From d94f40c805895ced3ad973ef99d0ee4ed1508d28 Mon Sep 17 00:00:00 2001 From: ti Date: Wed, 23 Apr 2025 17:39:02 +0200 Subject: [PATCH 3/6] Update Dockerfile to install necessary packages --- Dockerfile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 36ef8a7..988a428 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,8 @@ FROM pytorch/pytorch:2.3.1-cuda11.8-cudnn8-runtime - ARG user_name ARG uid ARG gid ENV HOME=/root - WORKDIR ${HOME} ENV DEBIAN_FRONTEND=noninteractive @@ -17,12 +15,25 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \ && apt-get clean \ && apt-get update +# Install conda packages as root +SHELL ["/bin/bash", "-lc"] +RUN conda install -y python=3.10 pytables=3.8.0 hdf5 jupyter + +# Install DeepLabCut and other pip packages as root +RUN pip install --no-cache-dir --upgrade pip setuptools && \ + pip install --no-cache-dir gpustat nvitop pytest PySide6==6.3.1 streamlit networkx isort black && \ + pip install --no-cache-dir amadeusgpt && \ + pip install --no-cache-dir git+https://github.com/DeepLabCut/DeepLabCut.git + +# Initialize conda for zsh shell +RUN /opt/conda/bin/conda init zsh + # Create a non-root user RUN mkdir /app /logs /data RUN groupadd -g ${gid} ${user_name} \ && useradd -m -u ${uid} -g ${gid} ${user_name} \ && chown -R ${uid}:${gid} /home -# + # Switch to the new user and set home directory as working directory USER ${user_name} ENV HOME=/home/${user_name} @@ -36,18 +47,7 @@ RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master RUN echo "export PATH=$PATH:/home/${user_name}/.local/bin" >> /home/${user_name}/.zshrc -# install pip packages -COPY ./conda/amadeusGPT.yml ${HOME}/amadeusGPT.yml -SHELL ["/bin/bash", "-lc"] -RUN source /opt/conda/etc/profile.d/conda.sh && \ - conda env create -f ${HOME}/amadeusGPT.yml && \ - conda activate amadeusgpt && \ - pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir gpustat nvitop pytest PySide6==6.3.1 streamlit networkx isort black git+https://github.com/DeepLabCut/DeepLabCut.git - -# USER ${user_name} USER root -# ENV HOME=/root WORKDIR /app CMD ["zsh"] From ddb8e8d4201a1c8c19851bf50962fc65dc703678 Mon Sep 17 00:00:00 2001 From: ti Date: Wed, 23 Apr 2025 17:42:16 +0200 Subject: [PATCH 4/6] - Added exec command to facilitate interactive access to the running container. --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 42de1c2..c4e32af 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ app: streamlit run amadeusgpt/app.py --server.fileWatcherType none --server.maxUploadSize 1000 - IMG_TAG := 0.1 IMG_NAME := amadeusgpt DOCKERFILE := Dockerfile @@ -25,10 +24,14 @@ DOCKER_SRC := /home/$(shell id -un)/AmadeusGPT VOLUMES := \ --volume $(HOST_SRC):$(DOCKER_SRC) -CONTAINER_TAG :=_test_v0.1 -CONTAINER_NAME := amadeusgpt_$(CONTAINER_TAG) # $(GPU) +CONTAINER_TAG :=v0.1 +CONTAINER_NAME := amadeusgpt_$(CONTAINER_TAG) run: docker run --shm-size=60G --gpus all -it --name $(CONTAINER_NAME) \ $(VOLUMES) \ - $(IMG_NAME):$(IMG_TAG) tail -f /dev/null \ No newline at end of file + $(IMG_NAME):$(IMG_TAG) +# tail -f /dev/null + +exec: + docker exec -it $(CONTAINER_NAME) /bin/bash From db5741007a9c5dd8836dd1f951c7932ca4d62514 Mon Sep 17 00:00:00 2001 From: ti Date: Thu, 24 Apr 2025 17:12:47 +0200 Subject: [PATCH 5/6] remove unnecessary packages; non-root user version (currently has permission denied bugs) --- Dockerfile | 52 +++++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) mode change 100644 => 100755 Dockerfile diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 index 988a428..73aa857 --- a/Dockerfile +++ b/Dockerfile @@ -1,54 +1,44 @@ +# Base image with PyTorch and CUDA support FROM pytorch/pytorch:2.3.1-cuda11.8-cudnn8-runtime + +# User arguments for container setup ARG user_name ARG uid ARG gid -ENV HOME=/root -WORKDIR ${HOME} +# Install system dependencies ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && apt-get install -y --no-install-recommends \ - vim zsh tmux wget curl htop jupyter python3 python3-pip libgl1-mesa-glx git sudo ssh libglib2.0-0 \ - tree \ + libgl1-mesa-glx libglib2.0-0 \ && apt-get -y autoclean \ && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && apt-get clean \ - && apt-get update - -# Install conda packages as root -SHELL ["/bin/bash", "-lc"] -RUN conda install -y python=3.10 pytables=3.8.0 hdf5 jupyter - -# Install DeepLabCut and other pip packages as root -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir gpustat nvitop pytest PySide6==6.3.1 streamlit networkx isort black && \ - pip install --no-cache-dir amadeusgpt && \ - pip install --no-cache-dir git+https://github.com/DeepLabCut/DeepLabCut.git - -# Initialize conda for zsh shell -RUN /opt/conda/bin/conda init zsh + && apt-get clean # Create a non-root user -RUN mkdir /app /logs /data +RUN mkdir -p /app /home/${user_name} RUN groupadd -g ${gid} ${user_name} \ && useradd -m -u ${uid} -g ${gid} ${user_name} \ - && chown -R ${uid}:${gid} /home + && chown -R ${uid}:${gid} /home/${user_name} # Switch to the new user and set home directory as working directory USER ${user_name} ENV HOME=/home/${user_name} WORKDIR ${HOME} -# Install Oh My Zsh and plugins -RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" --unattended \ - && git clone https://github.com/zsh-users/zsh-autosuggestions ~/.oh-my-zsh/custom/plugins/zsh-autosuggestions \ - && git clone https://github.com/zsh-users/zsh-syntax-highlighting ~/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting \ - && sed -i 's/^plugins=(.*)$/plugins=(git zsh-autosuggestions zsh-syntax-highlighting)/' ~/.zshrc +# Create a user conda environment to avoid permission issues +SHELL ["/bin/bash", "-c"] +RUN conda create -y --name amadeusGPT python=3.10 +ENV PATH=/opt/conda/envs/amadeusGPT/bin:$PATH +RUN conda install -y -n amadeusGPT hdf5 pytables=3.8.0 -RUN echo "export PATH=$PATH:/home/${user_name}/.local/bin" >> /home/${user_name}/.zshrc +# Install pip packages in the user environment +RUN pip install --no-cache-dir notebook amadeusgpt +RUN pip install -U --pre deeplabcut -USER root -WORKDIR /app +# Initialize conda for the user's bash +RUN conda init bash +RUN echo 'conda activate amadeusGPT' >> ${HOME}/.bashrc -CMD ["zsh"] -SHELL ["/bin/zsh", "-c"] \ No newline at end of file +# Default command when container starts (activating the conda env) +CMD ["bash", "-l"] \ No newline at end of file From 2ba90f7878ac043f838e083ff9d1ff3c31eb1748 Mon Sep 17 00:00:00 2001 From: ti Date: Thu, 24 Apr 2025 17:30:15 +0200 Subject: [PATCH 6/6] remove non-root user creation and switch to root for default command execution (this version past the test using notebooks/MausHaus_demo.ipynb) --- Dockerfile | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 73aa857..85e82e1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Base image with PyTorch and CUDA support FROM pytorch/pytorch:2.3.1-cuda11.8-cudnn8-runtime -# User arguments for container setup +# User arguments for container setup - keeping for compatibility with Makefile ARG user_name ARG uid ARG gid @@ -15,18 +15,6 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ && apt-get clean -# Create a non-root user -RUN mkdir -p /app /home/${user_name} -RUN groupadd -g ${gid} ${user_name} \ - && useradd -m -u ${uid} -g ${gid} ${user_name} \ - && chown -R ${uid}:${gid} /home/${user_name} - -# Switch to the new user and set home directory as working directory -USER ${user_name} -ENV HOME=/home/${user_name} -WORKDIR ${HOME} - -# Create a user conda environment to avoid permission issues SHELL ["/bin/bash", "-c"] RUN conda create -y --name amadeusGPT python=3.10 ENV PATH=/opt/conda/envs/amadeusGPT/bin:$PATH @@ -36,9 +24,11 @@ RUN conda install -y -n amadeusGPT hdf5 pytables=3.8.0 RUN pip install --no-cache-dir notebook amadeusgpt RUN pip install -U --pre deeplabcut -# Initialize conda for the user's bash +# Initialize conda for bash RUN conda init bash -RUN echo 'conda activate amadeusGPT' >> ${HOME}/.bashrc +RUN echo 'conda activate amadeusGPT' >> ~/.bashrc +USER root # Default command when container starts (activating the conda env) +WORKDIR /app CMD ["bash", "-l"] \ No newline at end of file