Repository navigation

Load workbench datasets into Blaze and HAPI with blazectl #235

Workflow file for this run

	name: CI

	on:
	push:
	branches:
	- main
	- release
	- develop
	tags:
	- v[0-9]+.[0-9]+**
	pull_request:
	branches:
	- main
	- release
	- develop

	env:
	TRIVY_VERSION: v0.75.0

	jobs:

	test:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Test Synthea import and roundtrip checks
	run: python3 -m unittest discover -s scripts/tests -v

	- name: Check Compose profile combinations
	run: python3 web/integration/test_compose_profiles.py

	- name: Test official availability updater
	run: >-
	docker run --rm --network none --memory=512m
	--entrypoint python
	-v "$PWD/web/integration/test_availability_layout.py:/test_layout.py:ro"
	ghcr.io/medizininformatik-initiative/dataportal-availability-updater:0.4.2
	/test_layout.py

	- name: Set up JDK 17
	uses: actions/setup-java@v3
	with:
	distribution: 'zulu'
	java-version: 17

	- name: Cache Local Maven Repo
	uses: actions/cache@v4
	with:
	path: ~/.m2/repository
	key: maven-repo

	- name: Initialize CodeQL
	uses: github/codeql-action/init@v3
	with:
	languages: java
	queries: security-and-quality

	- name: Build
	run: mvn -B package

	- name: Check converter option contract
	run: \|
	python3 web/catalog/options/generate_schema.py --check
	python3 -m unittest discover -s web/catalog/options/tests -v
	java --class-path target/excel2fhir.jar web/catalog/options/tests/CheckJavaBindings.java

	- name: Set up Python for workbench tests
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Test workbench queue and API
	run: \|
	python -m pip install -r web/backend/requirements-test.txt
	python -m unittest discover -s web/backend -p 'test_*.py' -v
	python web/integration/smoke_dataset_variants.py

	- name: Set up Node.js
	uses: actions/setup-node@v4
	with:
	node-version: 22

	- name: Test and build workbench
	working-directory: web/frontend
	run: \|
	npm ci
	npm test
	npm run build

	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v3

	workbench-inputs:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Build workbench worker
	run: docker build -f web/backend/Dockerfile --target worker -t excel2fhir-workbench:ci .
	- name: Verify blazectl uploads on Blaze and HAPI
	run: \|
	trap 'docker compose -f web/compose.yml stop blaze hapi hapi-db' EXIT
	docker compose -f web/compose.yml up -d --wait --wait-timeout 180 blaze
	docker run --rm --memory=1g --network excel2fhir-web-prototype_default -v "$PWD/web/integration/smoke_fhir_upload.py:/probe.py:ro" excel2fhir-workbench:ci python /probe.py blaze
	docker compose -f web/compose.yml stop blaze
	docker compose -f web/compose.yml up -d hapi-db hapi
	for attempt in $(seq 1 90); do
	if curl --fail --silent http://localhost:5191/fhir/metadata >/dev/null; then break; fi
	sleep 2
	done
	docker run --rm --memory=1g --network excel2fhir-web-prototype_default -v "$PWD/web/integration/smoke_fhir_upload.py:/probe.py:ro" excel2fhir-workbench:ci python /probe.py hapi
	- name: Generate Synthea sources and compare repeated clinical histories
	run: docker run --rm -i --memory=6g --cpus=2 excel2fhir-workbench:ci python - < web/integration/smoke_synthea_generation.py
	- name: Import Synthea sources and repeat immutable runs
	run: docker run --rm -i --memory=4g --cpus=2 excel2fhir-workbench:ci python - < web/integration/smoke_synthea_inputs.py

	security-scan:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Build Docker Image
	uses: docker/build-push-action@v4
	with:
	context: .
	file: docker/Dockerfile
	tags: security-scan-build:latest
	push: false

	- name: Run Trivy Vulnerability Scanner and Save to Sarif File
	uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
	with:
	version: ${{ env.TRIVY_VERSION }}
	image-ref: security-scan-build:latest
	format: sarif
	output: trivy-results.sarif
	severity: 'CRITICAL,HIGH'
	timeout: '15m0s'

	- name: Upload Trivy Scan Results to GitHub Security Tab
	uses: github/codeql-action/upload-sarif@v3
	with:
	sarif_file: trivy-results.sarif

	- name: Run Trivy Vulnerability Scanner
	uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
	with:
	version: ${{ env.TRIVY_VERSION }}
	image-ref: security-scan-build:latest
	exit-code: 1
	ignore-unfixed: true
	trivyignores: '.trivyignore'
	severity: 'CRITICAL,HIGH'
	timeout: '15m0s'

	synthea-workflow:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Build existing Synthea importer image
	run: docker build -f docker/synthea.Dockerfile -t excel2fhir-synthea:ci .
	- name: Convert and validate actual Synthea regression fixtures
	shell: bash
	run: \|
	mkdir -p "$RUNNER_TEMP/synthea-results"
	set +e
	docker run --rm --user "$(id -u):$(id -g)" -e HOME=/tmp \
	-v "$PWD/scripts/tests/fixtures/synthea:/input:ro" \
	-v "$RUNNER_TEMP/synthea-results:/output" \
	excel2fhir-synthea:ci -v -i /input -o /output
	result=$?
	set -e
	python3 - "$RUNNER_TEMP/synthea-results" "$result" <<'PYTHON'
	import json, pathlib, sys
	directory = next(pathlib.Path(sys.argv[1]).glob("run-*"))
	report = json.loads((directory / "details/reports/summary.json").read_text())
	assert not report['failures'], report
	assert len(report['results']) == 2, report
	assert report['status'] in ('COMPLETE', 'NOT_CHECKED'), report
	assert int(sys.argv[2]) == (0 if report['status'] == 'COMPLETE' else 1), report
	assert all(r['importStatus'] == 'COMPLETE' for r in report['results']), report
	bundles = [json.loads(p.read_text()) for p in sorted((directory / 'fhir').rglob('*.json'))]
	lines = [json.loads(line) for p in sorted((directory / 'fhir').rglob('*.ndjson')) for line in p.read_text().splitlines()]
	assert bundles == lines, 'JSON and NDJSON differ'
	assert not list((directory / 'details').glob('cases//run-/fhir/*/.json')), 'Duplicate final bundles'
	PYTHON
	- name: Scan Synthea importer image
	uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
	with:
	version: ${{ env.TRIVY_VERSION }}
	image-ref: excel2fhir-synthea:ci
	exit-code: 1
	ignore-unfixed: true
	trivyignores: '.trivyignore'
	severity: 'CRITICAL,HIGH'
	timeout: '15m0s'
	- name: Build generator and run the complete workflow offline
	shell: bash
	run: \|
	docker compose -p excel2fhir-ci -f compose.synthea.yml build
	docker run --rm --network none --entrypoint python3 excel2fhir-ci-synthea:latest \
	-m unittest discover -s /app/scripts/tests -p test_container_output.py -v
	set +e
	docker compose -p excel2fhir-ci -f compose.synthea.yml run --rm synthea
	result=$?
	set -e
	python3 - "$result" <<'PYTHON'
	import json, pathlib, sys
	runs = list(pathlib.Path('outputGlobal').glob('run-*'))
	assert len(runs) == 1, runs
	report = json.loads((runs[0] / 'details/reports/workflow.json').read_text())
	summary = json.loads((runs[0] / 'details/reports/summary.json').read_text())
	assert report['status'] == 'NOT_VALIDATED', report
	assert report['validationEnabled'] is False, report
	assert int(sys.argv[1]) == 0, report
	assert all(r['validationStatus'] == 'NOT_VALIDATED' for r in summary['results']), summary
	assert not list(runs[0].rglob('*.validation.json')), 'Unexpected validation reports'
	assert summary['results'] and not summary['failures'], summary
	bundles = [json.loads(p.read_text()) for p in sorted((runs[0] / 'fhir').rglob('*.json'))]
	lines = [json.loads(line) for p in sorted((runs[0] / 'fhir').rglob('*.ndjson')) for line in p.read_text().splitlines()]
	assert len(bundles) == len(summary['results'])
	assert bundles == lines, 'JSON and NDJSON differ'
	PYTHON
	- name: Scan complete workflow image
	uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
	with:
	version: ${{ env.TRIVY_VERSION }}
	image-ref: excel2fhir-ci-synthea:latest
	exit-code: 1
	ignore-unfixed: true
	trivyignores: '.trivyignore'
	severity: 'CRITICAL,HIGH'
	timeout: '15m0s'
	- name: Preserve regression reports and generated workbooks
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: synthea-workflow-results
	path: \|
	${{ runner.temp }}/synthea-results
	outputGlobal/

	build-excel2fhir:
	if: ${{ startsWith(github.ref, 'refs/tags/v') }}
	needs: [test, security-scan, synthea-workflow]
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Login to GitHub Docker Registry
	uses: docker/login-action@v2
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Prepare Version
	id: prep
	run: \|
	echo "repository=$(echo "${{ github.repository_owner }}" \| tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
	echo "version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT"

	- name: Build and push docker image for excel2fhir
	uses: docker/build-push-action@v4
	with:
	context: .
	file: docker/Dockerfile
	tags: \|
	ghcr.io/${{ steps.prep.outputs.repository }}/excel2fhir:latest
	ghcr.io/${{ steps.prep.outputs.repository }}/excel2fhir:${{ steps.prep.outputs.version }}
	push: true

Navigation Menu

Repository navigation

Load workbench datasets into Blaze and HAPI with blazectl #235

Workflow file

Load workbench datasets into Blaze and HAPI with blazectl #235

Uh oh!

Workflow file for this run