diff --git a/.github/workflows/ci-cd.yaml b/.github/workflows/ci-cd.yaml index 83946f7..d08e3a0 100644 --- a/.github/workflows/ci-cd.yaml +++ b/.github/workflows/ci-cd.yaml @@ -27,8 +27,10 @@ permissions: id-token: write env: # Define environment variables - ANTS_VERSION: v2.5.1 - ANTS_INSTALL_PATH: /opt/ants + SCIGET_NAME: ants + SCIGET_VERSION: 2.6.0 + # ANTS_VERSION: v2.5.1 + # ANTS_INSTALL_PATH: /opt/ants # Force tox and pytest to use color FORCE_COLOR: true jobs: @@ -56,47 +58,66 @@ jobs: steps: - - name: Install prerequisite packages - run: sudo apt install -y cmake - - name: Removed unnecessary tools to free space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Create ANTs install directory - run: | - sudo mkdir -p ${{ env.ANTS_INSTALL_PATH }} - sudo chown $USER ${{ env.ANTS_INSTALL_PATH }} + - name: Checkout repo + uses: actions/checkout@v4 - - name: Cache ANTs Install - id: cache-install - uses: actions/cache@v4 - with: - path: ${{ env.ANTS_INSTALL_PATH }} - key: ants-${{ env.ANTS_VERSION }}-${{ runner.os }} + - name: Install Apptainer and Lmod + run: | + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt-get update + sudo apt-get install -y lmod apptainer - - name: Install ANTs Package - if: steps.cache-install.outputs.cache-hit != 'true' + - name: Install Neurocommand run: | - pushd ~ - git clone https://github.com/ANTsX/ANTs.git - pushd ./ANTs - git checkout ${{ env.ANTS_VERSION }} - popd - mkdir build install - cd build - cmake -DCMAKE_INSTALL_PREFIX=${{ env.ANTS_INSTALL_PATH }} ../ANTs - make -j 4 - cd ANTS-build - make install - echo "Installation completed successfully" - echo "PATH=${{ env.ANTS_INSTALL_PATH }}/bin:$PATH" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ env.ANTS_INSTALL_PATH }}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + git clone https://github.com/NeuroDesk/neurocommand.git + pushd neurocommand + pip3 install -r neurodesk/requirements.txt --user + bash build.sh --cli + echo "APPTAINER_BINDPATH=`pwd -P`" >> $GITHUB_ENV + + source /etc/profile.d/lmod.sh + ./containers.sh "${{ env.SCIGET_NAME }}" | grep -E "^\s*\|\s*${{ env.SCIGET_NAME }}\s*\|\s*${{ env.SCIGET_VERSION }}\s*\|" | awk -F '|' '{gsub(/ /,"",$4); print $4}' + SCIGET_BUILDVERSION=$(./containers.sh "${{ env.SCIGET_NAME }}" | grep -E "^\s*\|\s*${{ env.SCIGET_NAME }}\s*\|\s*${{ env.SCIGET_VERSION }}\s*\|" | awk -F '|' '{gsub(/ /,"",$4); print $4}') + + ./local/fetch_containers.sh ${{ env.SCIGET_NAME }} ${{ env.SCIGET_VERSION }} $SCIGET_BUILDVERSION + echo "SCIGET_BUILDVERSION=$SCIGET_BUILDVERSION" >> $GITHUB_ENV popd - - name: Checkout repo - uses: actions/checkout@v4 + # - name: Create ANTs install directory + # run: | + # sudo mkdir -p ${{ env.ANTS_INSTALL_PATH }} + # sudo chown $USER ${{ env.ANTS_INSTALL_PATH }} + + # - name: Cache ANTs Install + # id: cache-install + # uses: actions/cache@v4 + # with: + # path: ${{ env.ANTS_INSTALL_PATH }} + # key: ants-${{ env.ANTS_VERSION }}-${{ runner.os }} + + # - name: Install ANTs Package + # if: steps.cache-install.outputs.cache-hit != 'true' + # run: | + # pushd ~ + # git clone https://github.com/ANTsX/ANTs.git + # pushd ./ANTs + # git checkout ${{ env.ANTS_VERSION }} + # popd + # mkdir build install + # cd build + # cmake -DCMAKE_INSTALL_PREFIX=${{ env.ANTS_INSTALL_PATH }} ../ANTs + # make -j 4 + # cd ANTS-build + # make install + # echo "Installation completed successfully" + # echo "PATH=${{ env.ANTS_INSTALL_PATH }}/bin:$PATH" >> $GITHUB_ENV + # echo "LD_LIBRARY_PATH=${{ env.ANTS_INSTALL_PATH }}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + # popd - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -110,7 +131,12 @@ jobs: - name: Show tox config run: tox c - name: Run tox - run: tox -v --exit-and-dump-after 1200 + run: | + source /etc/profile.d/lmod.sh + module use $PWD/neurocommand/local/containers/modules/ + module load ${{ env.SCIGET_NAME }}/${{ env.SCIGET_VERSION }} + tox -v --exit-and-dump-after 1200 + - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/nipype-auto-conv/generate b/nipype-auto-conv/generate index d0be32c..331ba10 100755 --- a/nipype-auto-conv/generate +++ b/nipype-auto-conv/generate @@ -4,6 +4,8 @@ set -e conv_dir=$(dirname $0) CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +AUTO_CONV_SUFFIX="-auto-conv" +REBASED_SUFFIX="-rebased-on-auto-conv" if ! git diff-index --quiet HEAD --; then echo "Current branch '$CURRENT_BRANCH' has uncommitted changes. Please commit or stash them before proceeding." @@ -12,24 +14,38 @@ fi echo "Automatically converting Nipype tasks to Pydra tasks..." -echo "Apply latest changes to nipype-auto-conv specs to auto-conv branch..." -git fetch origin auto-conv || echo "Wasn't able to fetch origin/auto-conv, please check your internet connection" -git checkout -b ${CURRENT_BRANCH}-auto-conv +echo "Create '$CURRENT_BRANCH$REBASED_SUFFIX' and '$CURRENT_BRANCH$AUTO_CONV_SUFFIX' branches if not present and switch to $CURRENT_BRANCH$AUTO_CONV_SUFFIX" +echo "NB: $CURRENT_BRANCH$AUTO_CONV_SUFFIX will contain the conversion specs from '$CURRENT_BRANCH' and the auto-converted tasks," +echo "wherease '$CURRENT_BRANCH$REBASED_SUFFIX' will be the changes in the current branch rebased on that" +if [ "$CURRENT_BRANCH" == "*$REBASED_SUFFIX"]; then + BASE_BRANCH=${CURRENT_BRANCH%"$REBASED_SUFFIX"}$AUTO_CONV_SUFFIX + REBASE_BRANCH=$CURRENT_BRANCH + git checkout $BASE_BRANCH + git reset --hard $CURRENT_BRANCH +else + REBASE_BRANCH=${CURRENT_BRANCH}$REBASED_SUFFIX + BASE_BRANCH=${CURRENT_BRANCH}$AUTO_CONV_SUFFIX + git checkout -b $REBASE_BRANCH + git checkout -b $BASE_BRANCH +fi + +echo "Apply nipype-auto-conv spec changes between current branch and the main auto-conv branch to the rebase branch..." +git fetch origin auto-conv git reset origin/auto-conv git add $conv_dir/specs git commit -m "Update auto-conv specs with latest changes from '$CURRENT_BRANCH'" || echo true -# Ignore any other changes outside the auto-conv directory +# Ignore any other changes outside the nipype-auto-conv/specs directory git reset --hard HEAD echo "Running nipype2pydra conversion..." nipype2pydra convert $conv_dir/specs $conv_dir/.. -echo "Committing converted tasks to ${CURRENT_BRANCH}-auto-conv..." +echo "Committing converted tasks to ${CURRENT_BRANCH}$AUTO_CONV_SUFFIX..." git add pydra/tasks/ants git commit -m "Auto-converted Nipype tasks to Pydra tasks" || echo true -echo "Rebasing '$CURRENT_BRANCH' to apply changes..." -git checkout $CURRENT_BRANCH -git rebase ${CURRENT_BRANCH}-auto-conv +echo "Rebasing '$REBASE_BRANCH' to apply changes..." +git checkout $REBASE_BRANCH +git rebase $BASE_BRANCH -echo "Successfully converted Nipype tasks to Pydra tasks and rebased '$CURRENT_BRANCH' on top of the 'auto-conv' branch." +echo "Successfully converted Nipype tasks to Pydra tasks and rebased manual edits over the top of them in '$REBASE_BRANCH'