ci: Fix linting getting skipped (#7043)

* ci: Fix linting getting skipped

* ci: Add list-files json

* ci: Why do we need a Docker container again?

* ci: Can't use a script if it doesn't exist yet

Co-authored-by: adamrtalbot <[email protected]>

* ci: Skip outputs if they're empty

* style: Run lsp on fastqc

* ci: Use files to skip extra steps

* ci(lint): Get the module name

nf-core/tools#3141

* ci: Add what shell to run on

* ci: Clean up the names in the filter step

* ci: Add hack to only lint the real changes

* ci: Add more to debug step

* ci(lint): Get fancy with map

* ci: One more time

* ci: Maybe?

* ci: results => result

* ci: Death by a thousand s's

* ci: Use Set

* ci: Use one at a time

* ci(gpu): Update step name

* ci: Remove clean workspace after run

* ci: Why does that break things?

* Revert "style: Run lsp on fastqc"

This reverts commit da64fce.

---------

Co-authored-by: adamrtalbot <[email protected]>

Author: edmundmiller

.github/actions/nf-test-action/action.yml

@@ -73,8 +73,8 @@ runs:
         SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
         SENTIEON_AUTH_MECH: "GitHub Actions - token"
         TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }}
-        NFT_WORKDIR: "~"
       run: |
+        NFT_WORKDIR=~ \
         nf-test test \
           --profile=${{ inputs.profile }} \
           --tap=test.tap \

.github/actions/self-hosted-cleanup/Dockerfile

@@ -43,13 +43,17 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       # Expose detected tags as 'modules' and 'workflows' output variables
-      paths: ${{ steps.list.outputs.components }}
+      paths: ${{ steps.outputs.outputs.components }}
       modules: ${{ steps.outputs.outputs.modules }}
       subworkflows: ${{ steps.outputs.outputs.subworkflows}}
       # Prod for version bumping
     steps:
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
-        uses: ./.github/actions/self-hosted-cleanup
+        run: |
+          ls -la ./
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+          ls -la ./
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
           fetch-depth: 0
@@ -70,7 +74,7 @@ jobs:
           echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
       - name: debug
         run: |
-          echo ${{ steps.filter.outputs.components }}
+          echo ${{ steps.outputs.outputs.components }}
           echo ${{ steps.outputs.outputs.modules }}
           echo ${{ steps.outputs.outputs.subworkflows }}
 

.github/actions/self-hosted-cleanup/README.md

@@ -50,9 +50,11 @@ jobs:
     name: nf-core-changes
     runs-on: ubuntu-latest
     outputs:
-      tags: ${{ steps.filter.outputs.changes }}
-      modules: ${{ steps.tags.outputs.modules }}
-      subworkflows: ${{ steps.tags.outputs.subworkflows }}
+      # https://github.com/dorny/paths-filter?tab=readme-ov-file#custom-processing-of-changed-files
+      modules: ${{ steps.filter.outputs.modules }}
+      modules_files: ${{ steps.module_names.outputs.result }}
+      subworkflows: ${{ steps.filter.outputs.subworkflows }}
+      subworkflows_files: ${{ steps.subworkflow_names.outputs.result }}
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
         with:
@@ -61,29 +63,46 @@ jobs:
       - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
         id: filter
         with:
-          filters: "tests/config/pytest_modules.yml"
+          filters: |
+            modules:
+              - 'modules/nf-core/**'
+              - 'tests/modules/nf-core/**'
+            subworkflows:
+              - 'subworkflows/nf-core/**'
+              - 'tests/subworkflows/nf-core/**'
           token: ""
+          list-files: "json"
 
-      - name: Fetch module tags
-        id: tags
-        run: |
-          echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT
-          echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT
+      - name: Get module name
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+        id: module_names
+        with:
+          script: |
+            return [...new Set(${{ steps.filter.outputs.modules_files }}.map(path => path.replace('modules/nf-core/', '').split('/').slice(0,2).filter(x => x !== 'main.nf').join('/')))];
+
+      - name: Get subworkflow name
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+        id: subworkflow_names
+        with:
+          script: |
+            return [...new Set(${{ steps.filter.outputs.subworkflows_files }}.map(path => path.replace('subworkflows/nf-core/', '').split('/').slice(0,2).filter(x => x !== 'main.nf').join('/')))];
 
       - name: debug
         run: |
-          echo ${{ steps.tags.outputs.modules }}
-          echo ${{ steps.tags.outputs.subworkflows }}
+          echo ${{ steps.filter.outputs.modules_files }}
+          echo ${{ steps.filter.outputs.subworkflows_files }}
+          echo ${{ steps.module_names.outputs.result }}
+          echo ${{ steps.subworkflow_names.outputs.result }}
 
   nf-core-lint-modules:
     runs-on: ${{ github.event.inputs.runners  || 'self-hosted' }}
     name: nf-core lint modules
     needs: nf-core-changes
-    if: ${{ (needs.nf-core-changes.outputs.modules != '[]') }}
+    if: ${{ (needs.nf-core-changes.outputs.modules == 'true') }}
     strategy:
       fail-fast: false
       matrix:
-        tags: "${{ fromJson(needs.nf-core-changes.outputs.modules) }}"
+        module: "${{ fromJson(needs.nf-core-changes.outputs.modules_files) }}"
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
@@ -114,18 +133,18 @@ jobs:
       - name: Install nf-core tools development version
         run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
 
-      - name: Lint module ${{ matrix.tags }}
-        run: nf-core modules lint ${{ matrix.tags }}
+      - name: Lint module ${{ matrix.module }}
+        run: nf-core modules lint ${{ matrix.module }}
 
   nf-core-lint-subworkflows:
     runs-on: ubuntu-latest
     name: nf-core lint subworkflows
     needs: nf-core-changes
-    if: ${{ (needs.nf-core-changes.outputs.subworkflows != '[]') }}
+    if: ${{ (needs.nf-core-changes.outputs.subworkflows == 'true') }}
     strategy:
       fail-fast: false
       matrix:
-        tags: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows) }}"
+        subworkflow: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows_files) }}"
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
@@ -148,8 +167,8 @@ jobs:
       - name: Install nf-core tools development version
         run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
 
-      - name: Lint module ${{ matrix.tags }}
-        run: nf-core subworkflows lint ${{ matrix.tags }}
+      - name: Lint subworkflow ${{ matrix.subworkflow }}
+        run: nf-core subworkflows lint ${{ matrix.subworkflow }}
 
   confirm-pass:
     runs-on: ubuntu-latest

.github/actions/self-hosted-cleanup/action.yml

@@ -63,7 +63,11 @@ jobs:
       SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
     steps:
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
-        uses: ./.github/actions/self-hosted-cleanup
+        run: |
+          ls -la ./
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+          ls -la ./
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
           fetch-depth: 0
@@ -73,5 +77,3 @@ jobs:
           profile: ${{ matrix.profile }}
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
-      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
-        uses: ./.github/actions/self-hosted-cleanup