ci.yml

name: CI

on:
  push:
    branches:
    - main
    - release
    - develop
    tags:
    - v[0-9]+.[0-9]+**
  pull_request:
    branches:
    - main
    - release
    - develop

env:
  TRIVY_VERSION: v0.75.0

jobs:

  test:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Test Synthea import and roundtrip checks
        run: python3 -m unittest discover -s scripts/tests -v

      - name: Check Compose profile combinations
        run: python3 web/integration/test_compose_profiles.py

      - name: Test official availability updater
        run: >-
          docker run --rm --network none --memory=512m
          --entrypoint python
          -v "$PWD/web/integration/test_availability_layout.py:/test_layout.py:ro"
          ghcr.io/medizininformatik-initiative/dataportal-availability-updater:0.4.2
          /test_layout.py

      - name: Set up JDK 17
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: 17

      - name: Cache Local Maven Repo
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: maven-repo

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: java
          queries: security-and-quality

      - name: Build
        run: mvn -B package

      - name: Check converter option contract
        run: |
          python3 web/catalog/options/generate_schema.py --check
          python3 -m unittest discover -s web/catalog/options/tests -v
          java --class-path target/excel2fhir.jar web/catalog/options/tests/CheckJavaBindings.java

      - name: Set up Python for workbench tests
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Test workbench queue and API
        run: |
          python -m pip install -r web/backend/requirements-test.txt
          python -m unittest discover -s web/backend -p 'test_*.py' -v
          python web/integration/smoke_dataset_variants.py

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Test and build workbench
        working-directory: web/frontend
        run: |
          npm ci
          npm test
          npm run build

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3

  workbench-inputs:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Build workbench worker
      run: docker build -f web/backend/Dockerfile --target worker -t excel2fhir-workbench:ci .
    - name: Verify blazectl uploads on Blaze and HAPI
      run: |
        trap 'docker compose -f web/compose.yml stop blaze hapi hapi-db' EXIT
        docker compose -f web/compose.yml up -d --wait --wait-timeout 180 blaze
        docker run --rm --memory=1g --network excel2fhir-web-prototype_default -v "$PWD/web/integration/smoke_fhir_upload.py:/probe.py:ro" excel2fhir-workbench:ci python /probe.py blaze
        docker compose -f web/compose.yml stop blaze
        docker compose -f web/compose.yml up -d hapi-db hapi
        for attempt in $(seq 1 90); do
          if curl --fail --silent http://localhost:5191/fhir/metadata >/dev/null; then break; fi
          sleep 2
        done
        docker run --rm --memory=1g --network excel2fhir-web-prototype_default -v "$PWD/web/integration/smoke_fhir_upload.py:/probe.py:ro" excel2fhir-workbench:ci python /probe.py hapi
    - name: Generate Synthea sources and compare repeated clinical histories
      run: docker run --rm -i --memory=6g --cpus=2 excel2fhir-workbench:ci python - < web/integration/smoke_synthea_generation.py
    - name: Import Synthea sources and repeat immutable runs
      run: docker run --rm -i --memory=4g --cpus=2 excel2fhir-workbench:ci python - < web/integration/smoke_synthea_inputs.py

  security-scan:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Build Docker Image
      uses: docker/build-push-action@v4
      with:
        context: .
        file: docker/Dockerfile
        tags: security-scan-build:latest
        push: false

    - name: Run Trivy Vulnerability Scanner and Save to Sarif File
      uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
      with:
        version: ${{ env.TRIVY_VERSION }}
        image-ref: security-scan-build:latest
        format: sarif
        output: trivy-results.sarif
        severity: 'CRITICAL,HIGH'
        timeout: '15m0s'

    - name: Upload Trivy Scan Results to GitHub Security Tab
      uses: github/codeql-action/upload-sarif@v3
      with:
        sarif_file: trivy-results.sarif

    - name: Run Trivy Vulnerability Scanner
      uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
      with:
        version: ${{ env.TRIVY_VERSION }}
        image-ref: security-scan-build:latest
        exit-code: 1
        ignore-unfixed: true
        trivyignores: '.trivyignore'
        severity: 'CRITICAL,HIGH'
        timeout: '15m0s'

  synthea-workflow:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Build existing Synthea importer image
      run: docker build -f docker/synthea.Dockerfile -t excel2fhir-synthea:ci .
    - name: Convert and validate actual Synthea regression fixtures
      shell: bash
      run: |
        mkdir -p "$RUNNER_TEMP/synthea-results"
        set +e
        docker run --rm --user "$(id -u):$(id -g)" -e HOME=/tmp \
          -v "$PWD/scripts/tests/fixtures/synthea:/input:ro" \
          -v "$RUNNER_TEMP/synthea-results:/output" \
          excel2fhir-synthea:ci -v -i /input -o /output
        result=$?
        set -e
        python3 - "$RUNNER_TEMP/synthea-results" "$result" <<'PYTHON'
        import json, pathlib, sys
        directory = next(pathlib.Path(sys.argv[1]).glob("run-*"))
        report = json.loads((directory / "details/reports/summary.json").read_text())
        assert not report['failures'], report
        assert len(report['results']) == 2, report
        assert report['status'] in ('COMPLETE', 'NOT_CHECKED'), report
        assert int(sys.argv[2]) == (0 if report['status'] == 'COMPLETE' else 1), report
        assert all(r['importStatus'] == 'COMPLETE' for r in report['results']), report
        bundles = [json.loads(p.read_text()) for p in sorted((directory / 'fhir').rglob('*.json'))]
        lines = [json.loads(line) for p in sorted((directory / 'fhir').rglob('*.ndjson')) for line in p.read_text().splitlines()]
        assert bundles == lines, 'JSON and NDJSON differ'
        assert not list((directory / 'details').glob('cases/*/run-*/fhir/**/*.json')), 'Duplicate final bundles'
        PYTHON
    - name: Scan Synthea importer image
      uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
      with:
        version: ${{ env.TRIVY_VERSION }}
        image-ref: excel2fhir-synthea:ci
        exit-code: 1
        ignore-unfixed: true
        trivyignores: '.trivyignore'
        severity: 'CRITICAL,HIGH'
        timeout: '15m0s'
    - name: Build generator and run the complete workflow offline
      shell: bash
      run: |
        docker compose -p excel2fhir-ci -f compose.synthea.yml build
        docker run --rm --network none --entrypoint python3 excel2fhir-ci-synthea:latest \
          -m unittest discover -s /app/scripts/tests -p test_container_output.py -v
        set +e
        docker compose -p excel2fhir-ci -f compose.synthea.yml run --rm synthea
        result=$?
        set -e
        python3 - "$result" <<'PYTHON'
        import json, pathlib, sys
        runs = list(pathlib.Path('outputGlobal').glob('run-*'))
        assert len(runs) == 1, runs
        report = json.loads((runs[0] / 'details/reports/workflow.json').read_text())
        summary = json.loads((runs[0] / 'details/reports/summary.json').read_text())
        assert report['status'] == 'NOT_VALIDATED', report
        assert report['validationEnabled'] is False, report
        assert int(sys.argv[1]) == 0, report
        assert all(r['validationStatus'] == 'NOT_VALIDATED' for r in summary['results']), summary
        assert not list(runs[0].rglob('*.validation.json')), 'Unexpected validation reports'
        assert summary['results'] and not summary['failures'], summary
        bundles = [json.loads(p.read_text()) for p in sorted((runs[0] / 'fhir').rglob('*.json'))]
        lines = [json.loads(line) for p in sorted((runs[0] / 'fhir').rglob('*.ndjson')) for line in p.read_text().splitlines()]
        assert len(bundles) == len(summary['results'])
        assert bundles == lines, 'JSON and NDJSON differ'
        PYTHON
    - name: Scan complete workflow image
      uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
      with:
        version: ${{ env.TRIVY_VERSION }}
        image-ref: excel2fhir-ci-synthea:latest
        exit-code: 1
        ignore-unfixed: true
        trivyignores: '.trivyignore'
        severity: 'CRITICAL,HIGH'
        timeout: '15m0s'
    - name: Preserve regression reports and generated workbooks
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: synthea-workflow-results
        path: |
          ${{ runner.temp }}/synthea-results
          outputGlobal/

  build-excel2fhir:
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    needs: [test, security-scan, synthea-workflow]
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Login to GitHub Docker Registry
      uses: docker/login-action@v2
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Prepare Version
      id: prep
      run: |
        echo "repository=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
        echo "version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT"

    - name: Build and push docker image for excel2fhir
      uses: docker/build-push-action@v4
      with:
        context: .
        file: docker/Dockerfile
        tags: |
          ghcr.io/${{ steps.prep.outputs.repository }}/excel2fhir:latest
          ghcr.io/${{ steps.prep.outputs.repository }}/excel2fhir:${{ steps.prep.outputs.version }}
        push: true