updated ci-cd to use neurocontainer for the ANTs package used in tests

.github/workflows/ci-cd.yaml

@@ -27,8 +27,10 @@ permissions:
   id-token: write
 
 env: # Define environment variables
-  ANTS_VERSION: v2.5.1
-  ANTS_INSTALL_PATH: /opt/ants
+  SCIGET_NAME: ants
+  SCIGET_VERSION: 2.6.0
+  # ANTS_VERSION: v2.5.1
+  # ANTS_INSTALL_PATH: /opt/ants
   # Force tox and pytest to use color
   FORCE_COLOR: true
 jobs:
@@ -56,47 +58,66 @@ jobs:
 
     steps:
 
-    - name: Install prerequisite packages
-      run: sudo apt install -y cmake
-
     - name: Removed unnecessary tools to free space
       run: |
         sudo rm -rf /usr/share/dotnet
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
-    - name: Create ANTs install directory
-      run: |
-        sudo mkdir -p ${{ env.ANTS_INSTALL_PATH }}
-        sudo chown $USER ${{ env.ANTS_INSTALL_PATH }}
+    - name: Checkout repo
+      uses: actions/checkout@v4
 
-    - name: Cache ANTs Install
-      id: cache-install
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.ANTS_INSTALL_PATH }}
-        key: ants-${{ env.ANTS_VERSION }}-${{ runner.os }}
+    - name: Install Apptainer and Lmod
+      run: |
+        sudo add-apt-repository -y ppa:apptainer/ppa
+        sudo apt-get update
+        sudo apt-get install -y lmod apptainer
 
-    - name: Install ANTs Package
-      if: steps.cache-install.outputs.cache-hit != 'true'
+    - name: Install Neurocommand
       run: |
-        pushd ~
-        git clone https://github.com/ANTsX/ANTs.git
-        pushd ./ANTs
-        git checkout ${{ env.ANTS_VERSION }}
-        popd
-        mkdir build install
-        cd build
-        cmake -DCMAKE_INSTALL_PREFIX=${{ env.ANTS_INSTALL_PATH }} ../ANTs
-        make -j 4
-        cd ANTS-build
-        make install
-        echo "Installation completed successfully"
-        echo "PATH=${{ env.ANTS_INSTALL_PATH }}/bin:$PATH" >> $GITHUB_ENV
-        echo "LD_LIBRARY_PATH=${{ env.ANTS_INSTALL_PATH }}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+        git clone https://github.com/NeuroDesk/neurocommand.git
+        pushd neurocommand
+        pip3 install -r neurodesk/requirements.txt --user
+        bash build.sh --cli
+        echo "APPTAINER_BINDPATH=`pwd -P`" >> $GITHUB_ENV
+
+        source /etc/profile.d/lmod.sh
+        ./containers.sh "${{ env.SCIGET_NAME }}" | grep -E "^\s*\|\s*${{ env.SCIGET_NAME }}\s*\|\s*${{ env.SCIGET_VERSION }}\s*\|" | awk -F '|' '{gsub(/ /,"",$4); print $4}'
+        SCIGET_BUILDVERSION=$(./containers.sh "${{ env.SCIGET_NAME }}" | grep -E "^\s*\|\s*${{ env.SCIGET_NAME }}\s*\|\s*${{ env.SCIGET_VERSION }}\s*\|" | awk -F '|' '{gsub(/ /,"",$4); print $4}')
+
+        ./local/fetch_containers.sh ${{ env.SCIGET_NAME }} ${{ env.SCIGET_VERSION }} $SCIGET_BUILDVERSION
+        echo "SCIGET_BUILDVERSION=$SCIGET_BUILDVERSION" >> $GITHUB_ENV
         popd
 
-    - name: Checkout repo
-      uses: actions/checkout@v4
+    # - name: Create ANTs install directory
+    #   run: |
+    #     sudo mkdir -p ${{ env.ANTS_INSTALL_PATH }}
+    #     sudo chown $USER ${{ env.ANTS_INSTALL_PATH }}
+
+    # - name: Cache ANTs Install
+    #   id: cache-install
+    #   uses: actions/cache@v4
+    #   with:
+    #     path: ${{ env.ANTS_INSTALL_PATH }}
+    #     key: ants-${{ env.ANTS_VERSION }}-${{ runner.os }}
+
+    # - name: Install ANTs Package
+    #   if: steps.cache-install.outputs.cache-hit != 'true'
+    #   run: |
+    #     pushd ~
+    #     git clone https://github.com/ANTsX/ANTs.git
+    #     pushd ./ANTs
+    #     git checkout ${{ env.ANTS_VERSION }}
+    #     popd
+    #     mkdir build install
+    #     cd build
+    #     cmake -DCMAKE_INSTALL_PREFIX=${{ env.ANTS_INSTALL_PATH }} ../ANTs
+    #     make -j 4
+    #     cd ANTS-build
+    #     make install
+    #     echo "Installation completed successfully"
+    #     echo "PATH=${{ env.ANTS_INSTALL_PATH }}/bin:$PATH" >> $GITHUB_ENV
+    #     echo "LD_LIBRARY_PATH=${{ env.ANTS_INSTALL_PATH }}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+    #     popd
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
@@ -110,7 +131,12 @@ jobs:
     - name: Show tox config
       run: tox c
     - name: Run tox
-      run: tox -v --exit-and-dump-after 1200
+      run: |
+        source /etc/profile.d/lmod.sh
+        module use $PWD/neurocommand/local/containers/modules/
+        module load ${{ env.SCIGET_NAME }}/${{ env.SCIGET_VERSION }}
+        tox -v --exit-and-dump-after 1200
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v5
       with:

nipype-auto-conv/generate

@@ -4,6 +4,8 @@ set -e
 conv_dir=$(dirname $0)
 
 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+AUTO_CONV_SUFFIX="-auto-conv"
+REBASED_SUFFIX="-rebased-on-auto-conv"
 
 if ! git diff-index --quiet HEAD --; then
     echo "Current branch '$CURRENT_BRANCH' has uncommitted changes. Please commit or stash them before proceeding."
@@ -12,24 +14,38 @@ fi
 
 echo "Automatically converting Nipype tasks to Pydra tasks..."
 
-echo "Apply latest changes to nipype-auto-conv specs to auto-conv branch..."
-git fetch origin auto-conv || echo "Wasn't able to fetch origin/auto-conv, please check your internet connection"
-git checkout -b ${CURRENT_BRANCH}-auto-conv
+echo "Create '$CURRENT_BRANCH$REBASED_SUFFIX' and '$CURRENT_BRANCH$AUTO_CONV_SUFFIX' branches if not present and switch to $CURRENT_BRANCH$AUTO_CONV_SUFFIX"
+echo "NB: $CURRENT_BRANCH$AUTO_CONV_SUFFIX will contain the conversion specs from '$CURRENT_BRANCH' and the auto-converted tasks,"
+echo "wherease '$CURRENT_BRANCH$REBASED_SUFFIX' will be the changes in the current branch rebased on that"
+if [ "$CURRENT_BRANCH" == "*$REBASED_SUFFIX"]; then
+    BASE_BRANCH=${CURRENT_BRANCH%"$REBASED_SUFFIX"}$AUTO_CONV_SUFFIX
+    REBASE_BRANCH=$CURRENT_BRANCH
+    git checkout $BASE_BRANCH
+    git reset --hard $CURRENT_BRANCH
+else
+    REBASE_BRANCH=${CURRENT_BRANCH}$REBASED_SUFFIX
+    BASE_BRANCH=${CURRENT_BRANCH}$AUTO_CONV_SUFFIX
+    git checkout -b $REBASE_BRANCH
+    git checkout -b $BASE_BRANCH
+fi
+
+echo "Apply nipype-auto-conv spec changes between current branch and the main auto-conv branch to the rebase branch..."
+git fetch origin auto-conv
 git reset origin/auto-conv
 git add $conv_dir/specs
 git commit -m "Update auto-conv specs with latest changes from '$CURRENT_BRANCH'" || echo true
-# Ignore any other changes outside the auto-conv directory
+# Ignore any other changes outside the nipype-auto-conv/specs directory
 git reset --hard HEAD
 
 echo "Running nipype2pydra conversion..."
 nipype2pydra convert $conv_dir/specs $conv_dir/..
 
-echo "Committing converted tasks to ${CURRENT_BRANCH}-auto-conv..."
+echo "Committing converted tasks to ${CURRENT_BRANCH}$AUTO_CONV_SUFFIX..."
 git add pydra/tasks/ants
 git commit -m "Auto-converted Nipype tasks to Pydra tasks" || echo true
 
-echo "Rebasing '$CURRENT_BRANCH' to apply changes..."
-git checkout $CURRENT_BRANCH
-git rebase ${CURRENT_BRANCH}-auto-conv
+echo "Rebasing '$REBASE_BRANCH' to apply changes..."
+git checkout $REBASE_BRANCH
+git rebase $BASE_BRANCH
 
-echo "Successfully converted Nipype tasks to Pydra tasks and rebased '$CURRENT_BRANCH' on top of the 'auto-conv' branch."
+echo "Successfully converted Nipype tasks to Pydra tasks and rebased manual edits over the top of them in '$REBASE_BRANCH'