Import scraped data to configured database #396
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Import scraped data to configured database | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [nightly-scrape-done] | |
| jobs: | |
| import_filings: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: "deploy" | |
| - name: Import candidate, PAC, and filing data | |
| run: | | |
| touch .env | |
| docker compose -f docker-compose.etl.yml run --rm \ | |
| -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e DATABASE_URL=${{ secrets.DATABASE_URL }} \ | |
| app make import/candidates import/pacs import/candidate_filings import/pac_filings | |
| import_transactions: | |
| runs-on: ubuntu-latest | |
| needs: import_filings | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| transaction_type: [CON, EXP] | |
| year: [2024, 2025, 2026] | |
| month: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: "deploy" | |
| - name: Import transaction data | |
| run: | | |
| touch .env | |
| docker compose -f docker-compose.etl.yml run --rm \ | |
| -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e DATABASE_URL=${{ secrets.DATABASE_URL }} \ | |
| app make import/${{ matrix.transaction_type }}_${{ matrix.month }}_${{ matrix.year }} |