From acdfb4015562aba48874f568811d05ba1321d19c Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 7 Jul 2025 18:40:07 +0100 Subject: [PATCH 1/8] Build studios compatible images --- .github/studios.Dockerfile | 58 +++++++++++++++++++++++ .github/workflows/docker-devcontainer.yml | 36 ++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 .github/studios.Dockerfile diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile new file mode 100644 index 000000000..f98d21693 --- /dev/null +++ b/.github/studios.Dockerfile @@ -0,0 +1,58 @@ +# Test Dockerfile for Studios-compatible Nextflow Training Container +# This uses the existing pre-built training container and adds Studios components + +# Add a default Connect client version +ARG CONNECT_CLIENT_VERSION="0.8" + +# Get the Seqera connect client +FROM public.cr.seqera.io/platform/connect-client:${CONNECT_CLIENT_VERSION} AS connect + +# Use the existing pre-built training container as base +FROM ghcr.io/nextflow-io/training:latest + +# Set default training tag version to match container version +ARG IMAGE_TAG=latest +ENV TRAINING_TAG=${IMAGE_TAG} + +# Install code-server and wget +RUN apt-get update && apt-get install -y wget && \ + curl -fsSL https://code-server.dev/install.sh | sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Copy the connect binary from the connect client image +COPY --from=connect /usr/bin/connect-client /usr/bin/connect-client + +# Install connect dependencies +RUN /usr/bin/connect-client --install && \ + /usr/bin/connect-client --setToolIdentifier vscode + +# Set up workspace environment +ENV HOME=/workspace \ + EDITOR=code \ + VISUAL=code \ + GIT_EDITOR="code --wait" \ + SHELL=bash + +# Create and configure workspace +WORKDIR /workspace +RUN mkdir -p /workspace/.nextflow && \ + mkdir -p /workspace/.vscode-server && \ + mkdir -p /workspace/training + +# Create setup script +RUN echo '#!/bin/bash\n\ +rm -rf /workspace/training/*\n\ +wget -qO- https://github.com/nextflow-io/training/archive/refs/tags/${TRAINING_TAG}.tar.gz | tar xz --strip-components=1 -C /workspace/training\n\ +nextflow help > /dev/null\n\ +exec "$@"' > /usr/local/bin/setup.sh && \ + chmod +x /usr/local/bin/setup.sh + +# Pre-heat nextflow +RUN nextflow help > /dev/null + +# Override the entrypoint to use connect-client +ENTRYPOINT ["/usr/bin/connect-client", "--entrypoint", "/usr/local/bin/setup.sh"] + +# Start code-server on the port specified by Studios, in the workspace directory +CMD ["/usr/bin/bash", "-c", "code-server --host 0.0.0.0 --port ${CONNECT_TOOL_PORT} --auth none --user-data-dir /workspace/.vscode-server /workspace/training"] diff --git a/.github/workflows/docker-devcontainer.yml b/.github/workflows/docker-devcontainer.yml index 5db2a6be1..19403a676 100644 --- a/.github/workflows/docker-devcontainer.yml +++ b/.github/workflows/docker-devcontainer.yml @@ -9,6 +9,7 @@ on: paths: - ".devcontainer/**" - ".github/workflows/docker-devcontainer.yml" + - ".github/studios.Dockerfile" push: { branches: [master] } # Build + push ('latest' tag) release: { types: [published] } # Build + push (release tag) workflow_dispatch: # Build + push (custom tag) @@ -68,3 +69,38 @@ jobs: push release workflow_dispatch + + build_studios: + needs: build_push + if: github.event_name != 'pull_request' || contains(github.event.pull_request.changed_files, '.github/studios.Dockerfile') + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Seqera Container Registry + uses: docker/login-action@v3 + with: + registry: cr.seqera.io + username: ${{ secrets.SCIDEV_REGISTRY_USER }} + password: ${{ secrets.SCIDEV_REGISTRY_PASSWORD }} + + - name: Build and push Studios image + uses: docker/build-push-action@v5 + with: + context: . + file: .github/studios.Dockerfile + push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64,linux/arm64 + tags: cr.seqera.io/scidev/nextflow-training-studios:${{ env.IMAGE_TAG }} + build-args: | + IMAGE_TAG=${{ env.IMAGE_TAG }} From d24a051805004149257ab1390b094d3d69b52a71 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 7 Jul 2025 18:41:52 +0100 Subject: [PATCH 2/8] Fix registry stuff --- .github/workflows/docker-devcontainer.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-devcontainer.yml b/.github/workflows/docker-devcontainer.yml index 19403a676..fc9136bf3 100644 --- a/.github/workflows/docker-devcontainer.yml +++ b/.github/workflows/docker-devcontainer.yml @@ -87,12 +87,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Seqera Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: - registry: cr.seqera.io - username: ${{ secrets.SCIDEV_REGISTRY_USER }} - password: ${{ secrets.SCIDEV_REGISTRY_PASSWORD }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Studios image uses: docker/build-push-action@v5 @@ -101,6 +101,6 @@ jobs: file: .github/studios.Dockerfile push: ${{ github.event_name != 'pull_request' }} platforms: linux/amd64,linux/arm64 - tags: cr.seqera.io/scidev/nextflow-training-studios:${{ env.IMAGE_TAG }} + tags: ghcr.io/${{ github.repository }}-studios:${{ env.IMAGE_TAG }} build-args: | IMAGE_TAG=${{ env.IMAGE_TAG }} From dc9fb174a115af50bf743fdd1f808baeca9631fe Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Tue, 29 Jul 2025 11:30:50 +0100 Subject: [PATCH 3/8] Refactor Studios Dockerfile for improved UX and stability - Switch to cr.seqera.io/public/data-studio-vscode base image - Change workspace path from /workspace to /workspaces (Codespaces convention) - Add comprehensive VS Code settings for dark mode, auto-open terminal, and training directory - Pre-install nf-core extension pack for immediate availability - Patch init script to open VS Code in training directory by default - Add conservative JVM tuning to prevent Nextflow core dumps - Improve setup script with better Nextflow initialization and timing - Use heredoc syntax for cleaner settings.json formatting - Add proper variable usage with ${workspaceFolder} for terminal directory --- .github/studios.Dockerfile | 94 +++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile index f98d21693..d24af76e3 100644 --- a/.github/studios.Dockerfile +++ b/.github/studios.Dockerfile @@ -1,58 +1,58 @@ -# Test Dockerfile for Studios-compatible Nextflow Training Container -# This uses the existing pre-built training container and adds Studios components -# Add a default Connect client version -ARG CONNECT_CLIENT_VERSION="0.8" +ARG CONNECT_CLIENT_VERSION="0.8-rc" -# Get the Seqera connect client -FROM public.cr.seqera.io/platform/connect-client:${CONNECT_CLIENT_VERSION} AS connect +FROM cr.seqera.io/public/data-studio-vscode:1.101.2-${CONNECT_CLIENT_VERSION} -# Use the existing pre-built training container as base -FROM ghcr.io/nextflow-io/training:latest - -# Set default training tag version to match container version -ARG IMAGE_TAG=latest +ARG IMAGE_TAG=2.2.1 ENV TRAINING_TAG=${IMAGE_TAG} -# Install code-server and wget -RUN apt-get update && apt-get install -y wget && \ - curl -fsSL https://code-server.dev/install.sh | sh && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Copy the connect binary from the connect client image -COPY --from=connect /usr/bin/connect-client /usr/bin/connect-client - -# Install connect dependencies -RUN /usr/bin/connect-client --install && \ - /usr/bin/connect-client --setToolIdentifier vscode - -# Set up workspace environment -ENV HOME=/workspace \ - EDITOR=code \ - VISUAL=code \ - GIT_EDITOR="code --wait" \ - SHELL=bash - # Create and configure workspace -WORKDIR /workspace -RUN mkdir -p /workspace/.nextflow && \ - mkdir -p /workspace/.vscode-server && \ - mkdir -p /workspace/training - -# Create setup script -RUN echo '#!/bin/bash\n\ -rm -rf /workspace/training/*\n\ -wget -qO- https://github.com/nextflow-io/training/archive/refs/tags/${TRAINING_TAG}.tar.gz | tar xz --strip-components=1 -C /workspace/training\n\ -nextflow help > /dev/null\n\ -exec "$@"' > /usr/local/bin/setup.sh && \ +WORKDIR /workspaces +RUN mkdir -p /workspaces/.nextflow && \ + mkdir -p /workspaces/.vscode-server && \ + mkdir -p /workspaces/training + +# Configure VS Code settings for dark mode and auto-open terminal +RUN mkdir -p /workspaces/training/.vscode +COPY < /dev/null 2>&1\n\ +sleep 2\n\ +nextflow info > /dev/null 2>&1\n\ +sleep 2\n\ +echo "Starting VS Code..."\n\ +exec /init' > /usr/local/bin/setup.sh && \ chmod +x /usr/local/bin/setup.sh -# Pre-heat nextflow -RUN nextflow help > /dev/null - # Override the entrypoint to use connect-client ENTRYPOINT ["/usr/bin/connect-client", "--entrypoint", "/usr/local/bin/setup.sh"] -# Start code-server on the port specified by Studios, in the workspace directory -CMD ["/usr/bin/bash", "-c", "code-server --host 0.0.0.0 --port ${CONNECT_TOOL_PORT} --auth none --user-data-dir /workspace/.vscode-server /workspace/training"] +# Default arguments (empty since setup script handles everything) +CMD [] From ad4a2ee6bac6dcaa56c7bb5661072035b32dd286 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Tue, 29 Jul 2025 11:45:05 +0100 Subject: [PATCH 4/8] Appease linter --- .github/studios.Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile index d24af76e3..535d9823c 100644 --- a/.github/studios.Dockerfile +++ b/.github/studios.Dockerfile @@ -16,16 +16,16 @@ RUN mkdir -p /workspaces/.nextflow && \ RUN mkdir -p /workspaces/training/.vscode COPY < Date: Fri, 8 Aug 2025 11:06:10 +0100 Subject: [PATCH 5/8] Tidy up, make repo configurable --- .github/studios.Dockerfile | 83 +++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile index 535d9823c..ddf33eaff 100644 --- a/.github/studios.Dockerfile +++ b/.github/studios.Dockerfile @@ -3,53 +3,62 @@ ARG CONNECT_CLIENT_VERSION="0.8-rc" FROM cr.seqera.io/public/data-studio-vscode:1.101.2-${CONNECT_CLIENT_VERSION} -ARG IMAGE_TAG=2.2.1 -ENV TRAINING_TAG=${IMAGE_TAG} +# Environment variable for repository URL with default +ENV REPO_URL="https://github.com/nextflow-io/training.git" +ENV REPO_REF="2.3.0" # Create and configure workspace WORKDIR /workspaces RUN mkdir -p /workspaces/.nextflow && \ - mkdir -p /workspaces/.vscode-server && \ - mkdir -p /workspaces/training + mkdir -p /workspaces/.vscode-server -# Configure VS Code settings for dark mode and auto-open terminal -RUN mkdir -p /workspaces/training/.vscode -COPY < /usr/local/bin/setup.sh <<'BASH' && \ + chmod +x /usr/local/bin/setup.sh +#!/bin/bash +set -euo pipefail + +echo "Cloning repository: ${REPO_URL}" + +# Derive workspace dir from repo name +REPO_NAME=$(basename "${REPO_URL}" .git) +WORKSPACE_DIR="/workspaces/${REPO_NAME}" + +# Clone the specified ref (branch or tag) +git clone --depth 1 --branch "${REPO_REF}" "${REPO_URL}" "${WORKSPACE_DIR}" + +# VS Code settings scoped to the repo +mkdir -p "${WORKSPACE_DIR}/.vscode" +cat > "${WORKSPACE_DIR}/.vscode/settings.json" << 'EOF' { - "workbench.colorTheme": "Default Dark Modern", - "terminal.integrated.defaultProfile.linux": "bash", - "terminal.integrated.cwd": "${workspaceFolder}", - "workbench.startupEditor": "none", - "workbench.panel.defaultLocation": "bottom", - "workbench.panel.opensMaximized": "never", - "terminal.integrated.showOnStartup": "always", - "cSpell.diagnosticLevel": "Hint", - "nextflow.java.home": "/usr", - "nextflow.debug": false + "workbench.colorTheme": "Default Dark Modern", + "terminal.integrated.defaultProfile.linux": "bash", + "terminal.integrated.cwd": "${workspaceFolder}", + "workbench.startupEditor": "none", + "workbench.panel.defaultLocation": "bottom", + "workbench.panel.opensMaximized": "never", + "terminal.integrated.hideOnStartup": "never", + "cSpell.diagnosticLevel": "Hint", + "nextflow.java.home": "/usr", + "nextflow.debug": false } EOF -# Pre-install nf-core extension pack -RUN /home/.vscode/bin/openvscode-server --install-extension nf-core.nf-core-extensionpack --force +# Default folder points at the cloned repo +sed -i "s|--default-folder '/workspace'|--default-folder '${WORKSPACE_DIR}'|g" /init -# Patch init script to use training directory as default folder -RUN sed -i 's|--default-folder '"'"'/workspace'"'"'|--default-folder '"'"'/workspaces/training'"'"'|g' /init - -# Create setup script that downloads training materials then runs init -RUN echo -e '#!/bin/bash\n\ -echo "Downloading training materials..."\n\ -wget -qO- https://github.com/nextflow-io/training/archive/refs/tags/${TRAINING_TAG}.tar.gz | tar xz --strip-components=1 -C /workspaces/training\n\ -echo "Initializing Nextflow..."\n\ -# Set conservative JVM flags for memory management only\n\ -export NXF_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=50.0"\n\ -# Pre-warm Nextflow\n\ -nextflow help > /dev/null 2>&1\n\ -sleep 2\n\ -nextflow info > /dev/null 2>&1\n\ -sleep 2\n\ -echo "Starting VS Code..."\n\ -exec /init' > /usr/local/bin/setup.sh && \ - chmod +x /usr/local/bin/setup.sh +# Pre-warm Nextflow +export NXF_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=50.0" +nextflow help > /dev/null 2>&1 +sleep 2 +nextflow info > /dev/null 2>&1 +sleep 2 + +exec /init +BASH # Override the entrypoint to use connect-client ENTRYPOINT ["/usr/bin/connect-client", "--entrypoint", "/usr/local/bin/setup.sh"] From 1cb038b229c9ff71a01ea711f230e871e2cb6918 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 8 Aug 2025 11:10:07 +0100 Subject: [PATCH 6/8] Fix whitespace --- .github/studios.Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile index ddf33eaff..b3eeeb5a4 100644 --- a/.github/studios.Dockerfile +++ b/.github/studios.Dockerfile @@ -34,16 +34,16 @@ git clone --depth 1 --branch "${REPO_REF}" "${REPO_URL}" "${WORKSPACE_DIR}" mkdir -p "${WORKSPACE_DIR}/.vscode" cat > "${WORKSPACE_DIR}/.vscode/settings.json" << 'EOF' { - "workbench.colorTheme": "Default Dark Modern", - "terminal.integrated.defaultProfile.linux": "bash", - "terminal.integrated.cwd": "${workspaceFolder}", - "workbench.startupEditor": "none", - "workbench.panel.defaultLocation": "bottom", - "workbench.panel.opensMaximized": "never", - "terminal.integrated.hideOnStartup": "never", - "cSpell.diagnosticLevel": "Hint", - "nextflow.java.home": "/usr", - "nextflow.debug": false + "workbench.colorTheme": "Default Dark Modern", + "terminal.integrated.defaultProfile.linux": "bash", + "terminal.integrated.cwd": "${workspaceFolder}", + "workbench.startupEditor": "none", + "workbench.panel.defaultLocation": "bottom", + "workbench.panel.opensMaximized": "never", + "terminal.integrated.hideOnStartup": "never", + "cSpell.diagnosticLevel": "Hint", + "nextflow.java.home": "/usr", + "nextflow.debug": false } EOF From 69df12c41e62ff277d5d1776a7168c54480af16d Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 8 Aug 2025 17:36:42 +0100 Subject: [PATCH 7/8] Fix container restart issue in setup script - check if repo exists before cloning --- .github/studios.Dockerfile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile index b3eeeb5a4..9f6e27009 100644 --- a/.github/studios.Dockerfile +++ b/.github/studios.Dockerfile @@ -27,8 +27,17 @@ echo "Cloning repository: ${REPO_URL}" REPO_NAME=$(basename "${REPO_URL}" .git) WORKSPACE_DIR="/workspaces/${REPO_NAME}" -# Clone the specified ref (branch or tag) -git clone --depth 1 --branch "${REPO_REF}" "${REPO_URL}" "${WORKSPACE_DIR}" +# Check if repository already exists +if [ -d "${WORKSPACE_DIR}/.git" ]; then + echo "Repository already exists at ${WORKSPACE_DIR}, skipping clone" + cd "${WORKSPACE_DIR}" + # Update to the specified ref if needed + git fetch origin "${REPO_REF}" || true + git checkout "${REPO_REF}" || true +else + # Clone the specified ref (branch or tag) + git clone --depth 1 --branch "${REPO_REF}" "${REPO_URL}" "${WORKSPACE_DIR}" +fi # VS Code settings scoped to the repo mkdir -p "${WORKSPACE_DIR}/.vscode" From 239f40f0688c9eb29dc738dbccdef66859cee02e Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 8 Aug 2025 17:42:37 +0100 Subject: [PATCH 8/8] Optimize container restart - skip all setup if repo exists, only run init --- .github/studios.Dockerfile | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/studios.Dockerfile b/.github/studios.Dockerfile index 9f6e27009..5347784fd 100644 --- a/.github/studios.Dockerfile +++ b/.github/studios.Dockerfile @@ -29,19 +29,14 @@ WORKSPACE_DIR="/workspaces/${REPO_NAME}" # Check if repository already exists if [ -d "${WORKSPACE_DIR}/.git" ]; then - echo "Repository already exists at ${WORKSPACE_DIR}, skipping clone" - cd "${WORKSPACE_DIR}" - # Update to the specified ref if needed - git fetch origin "${REPO_REF}" || true - git checkout "${REPO_REF}" || true + echo "Repository already exists at ${WORKSPACE_DIR}, skipping setup" else # Clone the specified ref (branch or tag) git clone --depth 1 --branch "${REPO_REF}" "${REPO_URL}" "${WORKSPACE_DIR}" -fi -# VS Code settings scoped to the repo -mkdir -p "${WORKSPACE_DIR}/.vscode" -cat > "${WORKSPACE_DIR}/.vscode/settings.json" << 'EOF' + # VS Code settings scoped to the repo + mkdir -p "${WORKSPACE_DIR}/.vscode" + cat > "${WORKSPACE_DIR}/.vscode/settings.json" << 'EOF' { "workbench.colorTheme": "Default Dark Modern", "terminal.integrated.defaultProfile.linux": "bash", @@ -56,15 +51,16 @@ cat > "${WORKSPACE_DIR}/.vscode/settings.json" << 'EOF' } EOF -# Default folder points at the cloned repo -sed -i "s|--default-folder '/workspace'|--default-folder '${WORKSPACE_DIR}'|g" /init + # Default folder points at the cloned repo + sed -i "s|--default-folder '/workspace'|--default-folder '${WORKSPACE_DIR}'|g" /init -# Pre-warm Nextflow -export NXF_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=50.0" -nextflow help > /dev/null 2>&1 -sleep 2 -nextflow info > /dev/null 2>&1 -sleep 2 + # Pre-warm Nextflow + export NXF_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=50.0" + nextflow help > /dev/null 2>&1 + sleep 2 + nextflow info > /dev/null 2>&1 + sleep 2 +fi exec /init BASH