From 1b46a7a449f312edba58487a6cb913076ff6ff96 Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Wed, 22 Jun 2022 08:55:05 -0300 Subject: [PATCH 01/10] Revert "only ci-tests" This reverts commit c919e6c416145def363f5f493c802cbdb263d9d7. --- .github/workflows/build_test_push_images.yml | 85 +++++++++++++++++++ .../workflows/test_containers_singularity.yml | 3 - spykingcircus/Dockerfile | 2 + tridesclous/Dockerfile | 2 + 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build_test_push_images.yml diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml new file mode 100644 index 0000000..9efc5a1 --- /dev/null +++ b/.github/workflows/build_test_push_images.yml @@ -0,0 +1,85 @@ +name: CI/CD + +on: + pull_request: + branches: [main] + types: [synchronize, opened, reopened, ready_for_review] + +jobs: + prepare-matrix-jobs: + name: Prepare Build and Test Jobs + runs-on: "ubuntu-latest" + outputs: + has_changed_files: ${{ steps.changed-files.outputs.any_changed }} + sorters: ${{ steps.get-changed-sorters.outputs.sorters }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v23 + with: + files: | + */Dockerfile + + - name: Prepare jobs matrix + id: get-changed-sorters + run: | + # sorters variable will be a JSON-like array + sorters="[" + + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do + # Removing /Dockerfile from string + parsed=${file%*/Dockerfile*} + + # Appending to final string + sorters+="\"${parsed}\", " + done + sorters=${sorters%*, *} + sorters+="]" + + echo ${sorters} + echo "::set-output name=sorters::${sorters}" + + build-and-test-image: + name: Build and Test (${{ matrix.sorter }}) + if: needs.prepare-matrix-jobs.outputs.has_changed_files == 'true' + needs: prepare-matrix-jobs + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # "macos-latest", "windows-latest" + os: ["ubuntu-latest", ] + sorter: ${{ fromJSON(needs.prepare-matrix-jobs.outputs.sorters) }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + python-version: 3.8 + + - name: Build ${{ matrix.sorter }} Docker image + run: | + cd ${{ matrix.sorter }} + bash build.sh + cd .. + + - uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.8.3 + + - name: Install dependencies & Test ${{ matrix.sorter }} image + run: | + if [[ ${{ matrix.sorter }} == 'tridesclous' ]]; then + pip install tridesclous + fi + pip install -r requirements_test.txt + pytest -sv tests/test_singularity_containers.py::test_${{ matrix.sorter }} + + - name: Push image + run: | + echo TO BE DONE + # cd ${{ matrix.sorter }} + # bash push.sh + # cd .. diff --git a/.github/workflows/test_containers_singularity.yml b/.github/workflows/test_containers_singularity.yml index cdc2545..19cac5d 100644 --- a/.github/workflows/test_containers_singularity.yml +++ b/.github/workflows/test_containers_singularity.yml @@ -2,9 +2,6 @@ name: Test sorter images in singularity on: workflow_dispatch: - pull_request: - branches: [main] - types: [synchronize, opened, reopened, ready_for_review] jobs: test-images: diff --git a/spykingcircus/Dockerfile b/spykingcircus/Dockerfile index 27e2b1e..eafe1b0 100644 --- a/spykingcircus/Dockerfile +++ b/spykingcircus/Dockerfile @@ -14,3 +14,5 @@ RUN apt-get update && apt-get install -y packagekit-gtk3-module libcanberra-gtk- RUN pip install scikit-build RUN pip install cmake>=3.18 RUN pip install spyking-circus==1.1.0 + +# test \ No newline at end of file diff --git a/tridesclous/Dockerfile b/tridesclous/Dockerfile index 03a3016..4992867 100644 --- a/tridesclous/Dockerfile +++ b/tridesclous/Dockerfile @@ -20,3 +20,5 @@ RUN pip install h5py RUN pip install loky packaging RUN pip install tridesclous==1.6.5 + +# test \ No newline at end of file From 14cbeeb5691889caa13b4f6713ba6da428d7cf37 Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Thu, 23 Jun 2022 10:17:57 -0300 Subject: [PATCH 02/10] test docker push --- .github/workflows/build_test_push_images.yml | 10 ++++++++-- tridesclous/Dockerfile | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml index 9efc5a1..70d0dae 100644 --- a/.github/workflows/build_test_push_images.yml +++ b/.github/workflows/build_test_push_images.yml @@ -77,9 +77,15 @@ jobs: pip install -r requirements_test.txt pytest -sv tests/test_singularity_containers.py::test_${{ matrix.sorter }} + - name: Docker login + env: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - name: Push image run: | - echo TO BE DONE + docker tag spikeinterface/{{ matrix.sorter }}:latest {{ env.DOCKER_USER }}/{{ matrix.sorter }}:latest + docker push {{ env.DOCKER_USER }}/{{ matrix.sorter }}:latest # cd ${{ matrix.sorter }} # bash push.sh - # cd .. diff --git a/tridesclous/Dockerfile b/tridesclous/Dockerfile index 4992867..95d7faa 100644 --- a/tridesclous/Dockerfile +++ b/tridesclous/Dockerfile @@ -21,4 +21,3 @@ RUN pip install loky packaging RUN pip install tridesclous==1.6.5 -# test \ No newline at end of file From 0050acb7a421fa44fecdd64f79c08095a617e69a Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Thu, 23 Jun 2022 10:29:29 -0300 Subject: [PATCH 03/10] Build push --- .github/workflows/build_test_push_images.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml index 70d0dae..ebb4cd9 100644 --- a/.github/workflows/build_test_push_images.yml +++ b/.github/workflows/build_test_push_images.yml @@ -77,15 +77,11 @@ jobs: pip install -r requirements_test.txt pytest -sv tests/test_singularity_containers.py::test_${{ matrix.sorter }} - - name: Docker login + - name: Docker login & Push image env: DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - name: Push image - run: | - docker tag spikeinterface/{{ matrix.sorter }}:latest {{ env.DOCKER_USER }}/{{ matrix.sorter }}:latest - docker push {{ env.DOCKER_USER }}/{{ matrix.sorter }}:latest - # cd ${{ matrix.sorter }} - # bash push.sh + cd ${{ matrix.sorter }} + bash push.sh From 708405eeac31bb0a8c7518876e32e06a854b1d06 Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Thu, 23 Jun 2022 11:40:29 -0300 Subject: [PATCH 04/10] add pull_request, clear test changes --- .github/workflows/test_containers_singularity.yml | 3 +++ spykingcircus/Dockerfile | 4 +--- tridesclous/Dockerfile | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_containers_singularity.yml b/.github/workflows/test_containers_singularity.yml index 19cac5d..cdc2545 100644 --- a/.github/workflows/test_containers_singularity.yml +++ b/.github/workflows/test_containers_singularity.yml @@ -2,6 +2,9 @@ name: Test sorter images in singularity on: workflow_dispatch: + pull_request: + branches: [main] + types: [synchronize, opened, reopened, ready_for_review] jobs: test-images: diff --git a/spykingcircus/Dockerfile b/spykingcircus/Dockerfile index eafe1b0..319c38c 100644 --- a/spykingcircus/Dockerfile +++ b/spykingcircus/Dockerfile @@ -13,6 +13,4 @@ RUN apt-get update && apt-get install -y libglib2.0-0 libgl1-mesa-glx RUN apt-get update && apt-get install -y packagekit-gtk3-module libcanberra-gtk-module libcanberra-gtk3-module RUN pip install scikit-build RUN pip install cmake>=3.18 -RUN pip install spyking-circus==1.1.0 - -# test \ No newline at end of file +RUN pip install spyking-circus==1.1.0 \ No newline at end of file diff --git a/tridesclous/Dockerfile b/tridesclous/Dockerfile index 95d7faa..03a3016 100644 --- a/tridesclous/Dockerfile +++ b/tridesclous/Dockerfile @@ -20,4 +20,3 @@ RUN pip install h5py RUN pip install loky packaging RUN pip install tridesclous==1.6.5 - From 5e998452502b64c95b9614c281401677d37dd95c Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Thu, 23 Jun 2022 12:10:38 -0300 Subject: [PATCH 05/10] Add files_ignore --- .github/workflows/build_test_push_images.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml index ebb4cd9..d072479 100644 --- a/.github/workflows/build_test_push_images.yml +++ b/.github/workflows/build_test_push_images.yml @@ -22,6 +22,14 @@ jobs: with: files: | */Dockerfile + files_ignore: | + hdsort-compiled/* + ironclust-compiled/* + kilosort_no_license/* + kilosort_with_license/* + spikeinterface-jupyterlab/* + tests/* + waveclus-compiled/* - name: Prepare jobs matrix id: get-changed-sorters From 6e849e8c794596f7c4deb945d2d89d007051cfaf Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Thu, 23 Jun 2022 12:13:38 -0300 Subject: [PATCH 06/10] blankline --- spykingcircus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spykingcircus/Dockerfile b/spykingcircus/Dockerfile index 319c38c..27e2b1e 100644 --- a/spykingcircus/Dockerfile +++ b/spykingcircus/Dockerfile @@ -13,4 +13,4 @@ RUN apt-get update && apt-get install -y libglib2.0-0 libgl1-mesa-glx RUN apt-get update && apt-get install -y packagekit-gtk3-module libcanberra-gtk-module libcanberra-gtk3-module RUN pip install scikit-build RUN pip install cmake>=3.18 -RUN pip install spyking-circus==1.1.0 \ No newline at end of file +RUN pip install spyking-circus==1.1.0 From 6d1f7a63d9fc28366d9fe011d394643f970d2a61 Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Fri, 24 Jun 2022 09:52:17 -0300 Subject: [PATCH 07/10] Update .github/workflows/test_containers_singularity.yml Co-authored-by: Alessio Buccino --- .github/workflows/test_containers_singularity.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_containers_singularity.yml b/.github/workflows/test_containers_singularity.yml index cdc2545..59fbca5 100644 --- a/.github/workflows/test_containers_singularity.yml +++ b/.github/workflows/test_containers_singularity.yml @@ -31,5 +31,4 @@ jobs: - name: Run test singularity containers run: | - pip install tridesclous pytest -sv tests/test_singularity_containers.py From b06571c481002c1756c73d0126476b21a82000f6 Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Mon, 27 Jun 2022 08:21:57 -0300 Subject: [PATCH 08/10] Renamed workflow, updated trigger --- .github/workflows/build_test_push_images.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml index d072479..46eb3ce 100644 --- a/.github/workflows/build_test_push_images.yml +++ b/.github/workflows/build_test_push_images.yml @@ -1,9 +1,9 @@ -name: CI/CD +name: Build, Test and Push changed Dockerfiles on: - pull_request: - branches: [main] - types: [synchronize, opened, reopened, ready_for_review] + push: + branches: + - main jobs: prepare-matrix-jobs: From e9bbf15816c8861d6c5db75b50f02368995d8ed1 Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Mon, 27 Jun 2022 08:37:57 -0300 Subject: [PATCH 09/10] remove cd --- .github/workflows/build_test_push_images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml index 46eb3ce..d826cfb 100644 --- a/.github/workflows/build_test_push_images.yml +++ b/.github/workflows/build_test_push_images.yml @@ -71,7 +71,6 @@ jobs: run: | cd ${{ matrix.sorter }} bash build.sh - cd .. - uses: eWaterCycle/setup-singularity@v7 with: From 28cc69a71a8403da4ede07dbb73f9117999c51bb Mon Sep 17 00:00:00 2001 From: Chuang Yu Min Date: Mon, 27 Jun 2022 08:58:37 -0300 Subject: [PATCH 10/10] Allow build and test on PRs --- .github/workflows/build_test_push_images.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_test_push_images.yml b/.github/workflows/build_test_push_images.yml index d826cfb..eed57e8 100644 --- a/.github/workflows/build_test_push_images.yml +++ b/.github/workflows/build_test_push_images.yml @@ -4,6 +4,9 @@ on: push: branches: - main + pull_request: + branches: [main] + types: [synchronize, opened, reopened, ready_for_review] jobs: prepare-matrix-jobs: @@ -85,6 +88,7 @@ jobs: pytest -sv tests/test_singularity_containers.py::test_${{ matrix.sorter }} - name: Docker login & Push image + if: github.ref == 'refs/heads/main' env: DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}