Repository navigation

Retrieve HBCD Data Dictionary #66

Workflow file for this run

.github/workflows/retrieve_data_dictionary.yml at 6baf566

	name: Retrieve HBCD Data Dictionary

	permissions:
	contents: write
	actions: write
	issues: write

	on:
	schedule:
	- cron: "0 0 * * 1" # Runs every Monday at midnight UTC
	workflow_dispatch: # Allows manual triggering

	jobs:
	retrieve-data:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.x"

	- name: Install dependencies
	run: pip install requests pandas pyyaml

	- name: Run script
	env:
	HBCD_USERNAME: ${{ secrets.HBCD_USERNAME }}
	HBCD_PASSWORD: ${{ secrets.HBCD_PASSWORD }}
	run: \|
	# Retry logic for LORIS API calls (up to 3 attempts)
	MAX_ATTEMPTS=3
	ATTEMPT=1

	while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
	echo "🔄 Attempt $ATTEMPT of $MAX_ATTEMPTS to retrieve LORIS data..."

	if python scripts/retrieve_script.py --username $HBCD_USERNAME --password $HBCD_PASSWORD --output_dir loris_data_dictionaries; then
	echo "✅ Successfully retrieved LORIS data on attempt $ATTEMPT"
	break
	else
	echo "❌ Attempt $ATTEMPT failed"

	if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
	echo "🚨 All $MAX_ATTEMPTS attempts failed. LORIS retrieval unsuccessful."
	exit 1
	else
	echo "⏳ Waiting 30 seconds before retry..."
	sleep 30
	ATTEMPT=$((ATTEMPT + 1))
	fi
	fi
	done

	- name: Smart change detection
	id: change_detection
	run: \|
	# Get the latest downloaded file
	LATEST_FILE=$(ls -t loris_data_dictionaries/hbcd_data_dictionary_*.csv \| head -1)
	echo "latest_file=$LATEST_FILE" >> $GITHUB_OUTPUT

	# Find the most recent existing file (excluding the just-downloaded one)
	EXISTING_FILES=$(ls -t loris_data_dictionaries/hbcd_data_dictionary_*.csv 2>/dev/null \| tail -n +2)

	if [ -z "$EXISTING_FILES" ]; then
	echo "No previous data dictionary found - treating as first run"
	echo "trigger_conversion=true" >> $GITHUB_OUTPUT
	echo "change_reason=First data dictionary download" >> $GITHUB_OUTPUT
	else
	PREVIOUS_FILE=$(echo "$EXISTING_FILES" \| head -1)
	echo "Comparing $PREVIOUS_FILE with $LATEST_FILE"

	# Run smart change detection
	if python scripts/detect_changes.py --old-file "$PREVIOUS_FILE" --new-file "$LATEST_FILE" --quiet; then
	echo "trigger_conversion=true" >> $GITHUB_OUTPUT
	echo "change_reason=Significant changes detected" >> $GITHUB_OUTPUT
	else
	echo "trigger_conversion=false" >> $GITHUB_OUTPUT
	echo "change_reason=No significant changes detected" >> $GITHUB_OUTPUT
	fi
	fi

	- name: Upload comparison report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: change-detection-report-${{ github.run_id }}
	path: \|
	logs/comparison_report_*.json
	logs/change_detection_*.log
	if-no-files-found: ignore

	- name: Commit new data dictionary
	id: commit_csv
	run: \|
	git config --global user.name 'github-actions[bot]'
	git config --global user.email 'github-actions[bot]@users.noreply.github.com'
	git add "${{ steps.change_detection.outputs.latest_file }}"

	# Check if there are staged changes
	if git diff --staged --quiet; then
	echo "📋 No new CSV changes to commit (file already exists)"
	echo "committed=false" >> $GITHUB_OUTPUT
	else
	git commit -m "Add HBCD data dictionary - $(date +%Y-%m-%d)

	Change detection result: ${{ steps.change_detection.outputs.change_reason }}
	Conversion triggered: ${{ steps.change_detection.outputs.trigger_conversion }}"
	git push
	echo "✅ CSV committed to repository"
	echo "committed=true" >> $GITHUB_OUTPUT
	fi

	- name: Trigger conversion workflow
	if: steps.change_detection.outputs.trigger_conversion == 'true'
	uses: actions/github-script@v7
	with:
	script: \|
	const maxAttempts = 3;
	let attempt = 1;
	let success = false;

	while (attempt <= maxAttempts && !success) {
	try {
	console.log(`🔄 Attempt ${attempt} of ${maxAttempts} to trigger conversion workflow...`);

	const result = await github.rest.actions.createWorkflowDispatch({
	owner: context.repo.owner,
	repo: context.repo.repo,
	workflow_id: 'automated_update.yml',
	ref: 'main',
	inputs: {
	data_dictionary_file: '${{ steps.change_detection.outputs.latest_file }}',
	manual_trigger: 'false'
	}
	});

	console.log('✅ Successfully triggered automated conversion workflow');
	console.log('Reason:', '${{ steps.change_detection.outputs.change_reason }}');
	console.log('Response status:', result.status);
	success = true;

	} catch (error) {
	console.error(`❌ Attempt ${attempt} failed:`, error.message);

	if (attempt === maxAttempts) {
	console.error('🚨 All attempts failed to trigger conversion workflow');
	core.setFailed(`Failed to trigger conversion workflow after ${maxAttempts} attempts: ${error.message}`);
	} else {
	console.log('⏳ Waiting 10 seconds before retry...');
	await new Promise(resolve => setTimeout(resolve, 10000));
	attempt++;
	}
	}
	}

	- name: Log no-change decision
	if: steps.change_detection.outputs.trigger_conversion == 'false'
	run: \|
	echo "ℹ️ No conversion triggered: ${{ steps.change_detection.outputs.change_reason }}"
	echo "The downloaded data dictionary does not contain significant changes."
	echo "Check the comparison report artifact for detailed analysis."

	- name: Notify on failure
	if: failure()
	uses: actions/github-script@v7
	with:
	script: \|
	const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

	await github.rest.issues.create({
	owner: context.repo.owner,
	repo: context.repo.repo,
	title: `🚨 LORIS Data Retrieval Failed - ${new Date().toISOString().split('T')[0]}`,
	body: `The automated LORIS data dictionary retrieval failed.\n\nWorkflow Run: ${runUrl}\nDate: ${new Date().toISOString()}\n\nCheck the workflow logs for details and re-run once the issue is resolved.`,
	labels: ['bug', 'automated']
	});

Navigation Menu

Repository navigation

Retrieve HBCD Data Dictionary #66

Workflow file

Retrieve HBCD Data Dictionary #66

Uh oh!

Workflow file for this run